[ Jun 26 05:34:16 Enabled. ] [ Jun 26 05:34:16 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/sled-agent/sled-agent run /opt/oxide/sled-agent/pkg/config.toml &"). ] [ Jun 26 05:34:16 Method "start" exited with status 0. ] [ Jun 26 05:34:16 Rereading configuration. ] [ Jun 26 05:34:16 No 'refresh' method defined. Treating as :true. ] note: configured to log to "/dev/stdout" {"msg":"Ensuring contract reaper thread","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:16.887872868Z","hostname":"buskin","pid":1555,"file":"illumos-utils/src/running_zone.rs:163"} {"msg":"finding links [\"net0\", \"net1\"]","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:16.888060613Z","hostname":"buskin","pid":1555,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:255"} {"msg":"Starting mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:16.941604092Z","hostname":"buskin","pid":1555,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:262","addr-objs":"[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]"} {"msg":"Importing mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:16.941653833Z","hostname":"buskin","pid":1555,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:42","path":"/opt/oxide/mg-ddm/pkg/ddm/manifest.xml"} {"msg":"Setting mg-ddm interfaces","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:17.054859154Z","hostname":"buskin","pid":1555,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:50","interfaces":"(\"net0/ll\" \"net1/ll\")"} {"msg":"Enabling mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:17.064341625Z","hostname":"buskin","pid":1555,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:56"} {"msg":"Ensuring zfs key directory exists","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:17.072588048Z","hostname":"buskin","pid":1555,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:272","path":"/var/run/oxide/"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:17.677725117Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fdb0:18c0:4d0d:9fb2::, len: 64 }"} {"msg":"Starting KeyManager","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:17.816643708Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:119"} {"msg":"Starting StorageManager","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:17.816679909Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:131"} {"msg":"Starting StorageMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:17.816772451Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:141"} {"msg":"Starting HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:17.816864574Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:162","nongimlet_observed_disks":"[]","sled_mode":"Scrimlet { asic: SoftNpuZone }"} {"msg":"DumpSetup poll loop started.","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:17.816912335Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:506"} {"msg":"Creating HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:17.816952745Z","hostname":"buskin","pid":1555,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:625"} {"msg":"Generated i86pc baseboard Pc { identifier: \"buskin\", model: \"i86pc\" }","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:17.819862153Z","hostname":"buskin","pid":1555,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:535"} {"msg":"Device is not a Gimlet (i86pc), proceeding with null hardware view","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:34:17.819869404Z","hostname":"buskin","pid":1555,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:674"} {"msg":"Starting HardwareMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:17.819888564Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:176"} {"msg":"Monitoring for hardware updates","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:17.819893324Z","hostname":"buskin","pid":1555,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:733"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:17.819918745Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/m2_0.vdev"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:17.819947326Z","hostname":"buskin","pid":1555,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:262","disks":"{}","underlay_network_info":"None"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:17.820024467Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:17.820059848Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:17.820074908Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/m2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:17.831580837Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:17.831614858Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:315","id":"968bbfe8-8122-4a85-96dd-954188a2f89d (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:17.928979739Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: 968bbfe8-8122-4a85-96dd-954188a2f89d (zpool), kind: Internal }"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.137541075Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.137582116Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/m2_1.vdev"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.137587596Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.137598956Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.137604016Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.137607896Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.137624927Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/m2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.146433672Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.146463653Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:315","id":"e8396e20-0493-4030-b55b-5c9e59a237cf (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.235363037Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: e8396e20-0493-4030-b55b-5c9e59a237cf (zpool), kind: Internal }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.440517574Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_0.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.440562935Z","hostname":"buskin","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 968bbfe8-8122-4a85-96dd-954188a2f89d (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: e8396e20-0493-4030-b55b-5c9e59a237cf (zpool), kind: Internal } }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.440585185Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.440596795Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.440601606Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.440606226Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.440612036Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_1.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.440650037Z","hostname":"buskin","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 968bbfe8-8122-4a85-96dd-954188a2f89d (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: e8396e20-0493-4030-b55b-5c9e59a237cf (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.440665827Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.440672577Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.440676668Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.440682088Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.440707488Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_2.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.440713088Z","hostname":"buskin","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 968bbfe8-8122-4a85-96dd-954188a2f89d (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: e8396e20-0493-4030-b55b-5c9e59a237cf (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.440726949Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.440731899Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.440735999Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.440740199Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.440744419Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_3.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.440748079Z","hostname":"buskin","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 968bbfe8-8122-4a85-96dd-954188a2f89d (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: e8396e20-0493-4030-b55b-5c9e59a237cf (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.440761089Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.440765939Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.440769719Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.44077349Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.44078121Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_4.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.4408066Z","hostname":"buskin","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 968bbfe8-8122-4a85-96dd-954188a2f89d (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: e8396e20-0493-4030-b55b-5c9e59a237cf (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.440826141Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.440830741Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.440834751Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.440838721Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.440842521Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_5.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.440849512Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.440852912Z","hostname":"buskin","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 968bbfe8-8122-4a85-96dd-954188a2f89d (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: e8396e20-0493-4030-b55b-5c9e59a237cf (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.440868732Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.440873492Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.440878762Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.440882132Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_6.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.440885992Z","hostname":"buskin","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 968bbfe8-8122-4a85-96dd-954188a2f89d (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: e8396e20-0493-4030-b55b-5c9e59a237cf (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.440913213Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.440918083Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.440922363Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.440925533Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.440929683Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_7.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.440935603Z","hostname":"buskin","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 968bbfe8-8122-4a85-96dd-954188a2f89d (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: e8396e20-0493-4030-b55b-5c9e59a237cf (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.440950384Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.440956394Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.440960784Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.440964804Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.440968404Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:232","vdev":"/scratch/u2_8.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.440979535Z","hostname":"buskin","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 968bbfe8-8122-4a85-96dd-954188a2f89d (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: e8396e20-0493-4030-b55b-5c9e59a237cf (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.440996255Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034 }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.441002495Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.441006335Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.441009615Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:494","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Waiting for boot disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.441014365Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:91"} {"msg":"Found boot disk DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.441025085Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:93"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.441028506Z","hostname":"buskin","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 968bbfe8-8122-4a85-96dd-954188a2f89d (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: e8396e20-0493-4030-b55b-5c9e59a237cf (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033 })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034 }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.441057696Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.441063017Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.441066686Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Starting Bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.441070267Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:200"} {"msg":"No ledger in /pool/int/968bbfe8-8122-4a85-96dd-954188a2f89d/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.441126738Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/e8396e20-0493-4030-b55b-5c9e59a237cf/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.441139058Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found. Loading Fsm::State as Uninitialized","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.441153819Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:83"} {"msg":"No ledger in /pool/int/968bbfe8-8122-4a85-96dd-954188a2f89d/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.441167299Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/e8396e20-0493-4030-b55b-5c9e59a237cf/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.441171979Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found for network config","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.441175929Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:146"} {"msg":"Starting Bootstore DDMD poller","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.44120281Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:205"} {"msg":"Starting ZoneBundler related tasks","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.44120751Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:220"} {"msg":"Creating ServiceManager","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.452396821Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:710"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.452431082Z","hostname":"buskin","pid":1555,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:289","time_to_next_cleanup":"599.999957339s","next_cleanup":"Instant { tv_sec: 1182, tv_nsec: 632304199 }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.452488863Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.452497243Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3547"} {"msg":"No ledger in /pool/int/968bbfe8-8122-4a85-96dd-954188a2f89d/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.452540184Z","hostname":"buskin","pid":1555,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/e8396e20-0493-4030-b55b-5c9e59a237cf/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.452551705Z","hostname":"buskin","pid":1555,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.455443232Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:196"} {"msg":"Started listening","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.455557875Z","hostname":"buskin","pid":1555,"component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:44","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Injecting RSS configuration: RackInitializeRequest {\n trust_quorum_peers: None,\n bootstrap_discovery: OnlyOurs,\n ntp_servers: [\n \"0.pool.ntp.org\",\n ],\n dns_servers: [\n 1.1.1.1,\n 9.9.9.9,\n ],\n internal_services_ip_pool_ranges: [\n V4(\n Ipv4Range {\n first: 10.151.1.164,\n last: 10.151.1.173,\n },\n ),\n ],\n external_dns_ips: [\n 10.151.1.164,\n 10.151.1.165,\n ],\n external_dns_zone_name: \"oxide.test\",\n external_certificates: \"\",\n recovery_silo: RecoverySiloConfig {\n silo_name: Name(\n \"recovery\",\n ),\n user_name: UserId(\n \"recovery\",\n ),\n user_password_hash: NewPasswordHash(\n PasswordHashString {\n string: \"$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY\",\n encoding: B64,\n },\n ),\n },\n rack_network_config: RackNetworkConfigV2 {\n rack_subnet: Ipv6Net {\n addr: fd00:1122:3344:100::,\n width: 56,\n },\n infra_ip_first: 10.151.1.174,\n infra_ip_last: 10.151.1.174,\n ports: [\n PortConfigV2 {\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n nexthop: 10.151.1.100,\n vlan_id: None,\n },\n ],\n addresses: [\n UplinkAddressConfig {\n address: V4(\n Ipv4Net {\n addr: 10.151.1.174,\n width: 24,\n },\n ),\n vlan_id: None,\n },\n ],\n switch: Switch0,\n port: \"qsfp0\",\n uplink_port_speed: Speed40G,\n uplink_port_fec: None,\n bgp_peers: [],\n autoneg: false,\n },\n ],\n bgp: [],\n bfd: [],\n },\n allowed_source_ips: Any,\n}","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.45580526Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1011"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.455821621Z","hostname":"buskin","pid":1555,"component":"DnsResolver","component":"RSS","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.455884782Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/968bbfe8-8122-4a85-96dd-954188a2f89d/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.455931983Z","hostname":"buskin","pid":1555,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/e8396e20-0493-4030-b55b-5c9e59a237cf/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.455942574Z","hostname":"buskin","pid":1555,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"RSS configuration has not been fully applied yet","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.455949284Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1071"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.455983705Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/968bbfe8-8122-4a85-96dd-954188a2f89d/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.455992025Z","hostname":"buskin","pid":1555,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/e8396e20-0493-4030-b55b-5c9e59a237cf/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.456014475Z","hostname":"buskin","pid":1555,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"Creating new allocation plan","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.456019766Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1112"} {"msg":"Serializing plan","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.456023326Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:156"} {"msg":"Creating plan for the sled at fdb0:18c0:4d0d:9fb2::1","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.456026515Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:129"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.456029976Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Writing ledger to /pool/int/968bbfe8-8122-4a85-96dd-954188a2f89d/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.456054796Z","hostname":"buskin","pid":1555,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/e8396e20-0493-4030-b55b-5c9e59a237cf/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.456412035Z","hostname":"buskin","pid":1555,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Sled plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.456580679Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:176"} {"msg":"Writing Rack Network Configuration to bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.456603269Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1145"} {"msg":"Attempting to update network config with generation: 1, current_generation: 0","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.45665274Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/968bbfe8-8122-4a85-96dd-954188a2f89d/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.456663481Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/e8396e20-0493-4030-b55b-5c9e59a237cf/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.456838165Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 1","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.456971888Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"Received initialization request from RSS","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.457010878Z","hostname":"buskin","pid":1555,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:217","target_sled":"[fdb0:18c0:4d0d:9fb2::1]:12346","request":"StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: 8ead6dd9-9765-4776-9768-713d8773d37f, rack_id: f0fff194-33a7-4fc7-ac24-138176f3dba5, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }"} {"msg":"Accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.457132151Z","hostname":"buskin","pid":1555,"remote_addr":"[fdb0:18c0:4d0d:9fb2::1]:55290","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:66"} {"msg":"Loading Sled Agent: StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: 8ead6dd9-9765-4776-9768-713d8773d37f, rack_id: f0fff194-33a7-4fc7-ac24-138176f3dba5, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.457230194Z","hostname":"buskin","pid":1555,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:360"} {"msg":"KeyManager: using hardcoded secret retriever","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.457240334Z","hostname":"buskin","pid":1555,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:375"} {"msg":"setting up sled agent server","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.457262664Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/server.rs:44"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.457275835Z","hostname":"buskin","pid":1555,"component":"DnsResolver","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"Received KeyManagerReady","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.457282485Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Loading storage config","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.457286615Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:530"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.457290065Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:101::, len: 64 }"} {"msg":"No ledger in /pool/int/968bbfe8-8122-4a85-96dd-954188a2f89d/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.457336186Z","hostname":"buskin","pid":1555,"request":"load_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/e8396e20-0493-4030-b55b-5c9e59a237cf/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.457352486Z","hostname":"buskin","pid":1555,"request":"load_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger of physical disks exists","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.457358507Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:519"} {"msg":"KeyManager ready, but no ledger detected","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.457363827Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:547"} {"msg":"Enabling switch zone (new)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.460452099Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3779"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.46951328Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.479646757Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.489499007Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.498603809Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.506882182Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.514568961Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.522282201Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.52995658Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Zpool does not exist","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.537557468Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:603","err":"ZpoolDoesNotExist","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"No disks to be automatically managed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.537592229Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:558"} {"msg":"automatically managing disks","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.537596379Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:370"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.537600309Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:339"} {"msg":"Successfully synchronized disks without error","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.537604529Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:449","result":"DisksManagementResult { status: [] }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.537613239Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"SledAgent::new(..) starting","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.553430798Z","hostname":"buskin","pid":1555,"sled_id":"8ead6dd9-9765-4776-9768-713d8773d37f","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:354"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.553478079Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.55350591Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Requested swap device of size 64 GiB","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.55351931Z","hostname":"buskin","pid":1555,"sled_id":"8ead6dd9-9765-4776-9768-713d8773d37f","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:366"} {"msg":"Failed enable ddm stats (will retry after 107.43517ms","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.556340736Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"980d80ec-87de-4a1c-bf04-d24effd3b620\", \"content-length\": \"124\", \"date\": \"Wed, 26 Jun 2024 05:34:17 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"980d80ec-87de-4a1c-bf04-d24effd3b620\" }"} {"msg":"Configuring new Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.557598236Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"attempting to create encrypted zvol: name=\"oxi_968bbfe8-8122-4a85-96dd-954188a2f89d/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.566413041Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/swap_device.rs:153"} {"msg":"Installing Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.576504766Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"successfully created encrypted zvol: name=\"oxi_968bbfe8-8122-4a85-96dd-954188a2f89d/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.589013909Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/swap_device.rs:249"} {"msg":"adding swap device: swapname=\"/dev/zvol/dsk/oxi_968bbfe8-8122-4a85-96dd-954188a2f89d/swap\"","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.58906155Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/swap_device.rs:96"} {"msg":"Mounting backing filesystems","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.593856131Z","hostname":"buskin","pid":1555,"sled_id":"8ead6dd9-9765-4776-9768-713d8773d37f","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:381"} {"msg":"Processing fmd","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.593866362Z","hostname":"buskin","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:126"} {"msg":"Ensuring dataset oxi_968bbfe8-8122-4a85-96dd-954188a2f89d/backing/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.593871562Z","hostname":"buskin","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:136"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.664727335Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Stopping service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.664863988Z","hostname":"buskin","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:170"} {"msg":"Failed enable ddm stats (will retry after 137.20498ms","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.667249964Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"eaf23b3d-6a2b-433d-9664-24f787c1a7e4\", \"content-length\": \"124\", \"date\": \"Wed, 26 Jun 2024 05:34:17 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"eaf23b3d-6a2b-433d-9664-24f787c1a7e4\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.805393567Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 132.510338ms","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.80595539Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"58b3e53b-1906-4367-9425-e30d0ffcb9d3\", \"content-length\": \"124\", \"date\": \"Wed, 26 Jun 2024 05:34:18 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"58b3e53b-1906-4367-9425-e30d0ffcb9d3\" }"} {"msg":"Mounting oxi_968bbfe8-8122-4a85-96dd-954188a2f89d/backing/fmd on /var/fm/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.883152201Z","hostname":"buskin","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:178"} {"msg":"Ensuring directory /var/fm/fmd/rsrc","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.896544444Z","hostname":"buskin","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Ensuring directory /var/fm/fmd/ckpt","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.896643336Z","hostname":"buskin","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Ensuring directory /var/fm/fmd/xprt","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.896676127Z","hostname":"buskin","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Starting service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.896692637Z","hostname":"buskin","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:192"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.939807173Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 240.692682ms","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:18.940343076Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ab03bb3c-9143-4e76-bb52-47eeb9907706\", \"content-length\": \"124\", \"date\": \"Wed, 26 Jun 2024 05:34:18 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"ab03bb3c-9143-4e76-bb52-47eeb9907706\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:19.182141777Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 159.290579ms","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:19.182912385Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"8bfcf46a-261b-429b-8e17-2f96c5a10b47\", \"content-length\": \"124\", \"date\": \"Wed, 26 Jun 2024 05:34:18 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"8bfcf46a-261b-429b-8e17-2f96c5a10b47\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:19.344017834Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 305.657783ms","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:19.344601408Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:178","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c1474062-7911-4a6f-84d3-7c1620c53740\", \"content-length\": \"124\", \"date\": \"Wed, 26 Jun 2024 05:34:18 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"c1474062-7911-4a6f-84d3-7c1620c53740\" }"} {"msg":"using '[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]' as data links for xde driver","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:19.62115325Z","hostname":"buskin","pid":1555,"sled_id":"8ead6dd9-9765-4776-9768-713d8773d37f","component":"SledAgent","file":"illumos-utils/src/opte/illumos.rs:90"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:19.621761614Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:42379","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:196","component":"producer-server","component":"MetricsManager","sled_id":"8ead6dd9-9765-4776-9768-713d8773d37f","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:196"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:19.621786815Z","hostname":"buskin","pid":1555,"component":"internal-dns-resolver","file":"internal-dns/src/resolver.rs:60","component":"producer-server","component":"MetricsManager","sled_id":"8ead6dd9-9765-4776-9768-713d8773d37f","component":"SledAgent","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"starting oximeter metric producer server","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:19.621856207Z","hostname":"buskin","pid":1555,"file":"oximeter/producer/src/lib.rs:282","component":"producer-server","component":"MetricsManager","sled_id":"8ead6dd9-9765-4776-9768-713d8773d37f","component":"SledAgent","file":"oximeter/producer/src/lib.rs:282","interval":"30s","address":"[fd00:1122:3344:101::1]:42379","producer_id":"8ead6dd9-9765-4776-9768-713d8773d37f"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:34:19.628733557Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:390","component":"producer-server","component":"MetricsManager","sled_id":"8ead6dd9-9765-4776-9768-713d8773d37f","component":"SledAgent","file":"oximeter/producer/src/lib.rs:390","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"366.285314ms"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:19.651441467Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:164","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"50% of 102952026112 physical ram = 51476013056 bytes)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:19.670146343Z","hostname":"buskin","pid":1555,"component":"VmmReservoirManager","sled_id":"8ead6dd9-9765-4776-9768-713d8773d37f","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:267"} {"msg":"Setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:19.670180784Z","hostname":"buskin","pid":1555,"component":"VmmReservoirManager","sled_id":"8ead6dd9-9765-4776-9768-713d8773d37f","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:275"} {"msg":"sled agent started","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:19.670283067Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:893","underlay_address":"fd00:1122:3344:101::1"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:19.670290477Z","hostname":"buskin","pid":1555,"component":"DnsResolver","component":"ServiceManager","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:19.671055915Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:196"} {"msg":"Sled Agent loaded; recording configuration","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:19.671095485Z","hostname":"buskin","pid":1555,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:430"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:19.671123376Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Writing ledger to /pool/int/968bbfe8-8122-4a85-96dd-954188a2f89d/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:19.671148627Z","hostname":"buskin","pid":1555,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/e8396e20-0493-4030-b55b-5c9e59a237cf/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:19.671362892Z","hostname":"buskin","pid":1555,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Sled Agent Started","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:19.671529786Z","hostname":"buskin","pid":1555,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:138"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:19.671559856Z","hostname":"buskin","pid":1555,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:262","disks":"{}","underlay_network_info":"Some((fd00:1122:3344:101::2, Some(RackNetworkConfigV2 { rack_subnet: Ipv6Net { addr: fd00:1122:3344:100::, width: 56 }, infra_ip_first: 10.151.1.174, infra_ip_last: 10.151.1.174, ports: [PortConfigV2 { routes: [RouteConfig { destination: V4(Ipv4Net { addr: 0.0.0.0, width: 0 }), nexthop: 10.151.1.100, vlan_id: None }], addresses: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }], switch: Switch0, port: \"qsfp0\", uplink_port_speed: Speed40G, uplink_port_fec: None, bgp_peers: [], autoneg: false }], bgp: [], bfd: [] })))"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:19.671575437Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3547"} {"msg":"Connection closed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:19.671579227Z","hostname":"buskin","pid":1555,"remote_addr":"[fdb0:18c0:4d0d:9fb2::1]:55290","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:73"} {"msg":"Peer agent initialized","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:19.846456727Z","hostname":"buskin","pid":1555,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:108","peer":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Initialized sled agent","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:19.846531549Z","hostname":"buskin","pid":1555,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:232","target_sled":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:19.846783855Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/968bbfe8-8122-4a85-96dd-954188a2f89d/config/rss-service-plan-v3.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:19.846895907Z","hostname":"buskin","pid":1555,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/e8396e20-0493-4030-b55b-5c9e59a237cf/config/rss-service-plan-v3.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:19.846907108Z","hostname":"buskin","pid":1555,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:19.846935288Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:19.846942959Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/8ead6dd9-9765-4776-9768-713d8773d37f): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:34:19.852654422Z","hostname":"buskin","pid":1555,"component":"NexusNotifierTask","sled_id":"8ead6dd9-9765-4776-9768-713d8773d37f","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:19.878461604Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:57596"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:19.885393556Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:19.885478127Z","hostname":"buskin","pid":1555,"uri":"/inventory","method":"GET","req_id":"96a7a649-faee-4468-8b9d-596d3168a665","remote_addr":"[fd00:1122:3344:101::1]:57596","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":150,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:19.913895321Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:54253"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:19.913959842Z","hostname":"buskin","pid":1555,"uri":"/sled-role","method":"GET","req_id":"49063325-5cd0-465f-bd2b-0f818266dc7f","remote_addr":"[fd00:1122:3344:101::1]:54253","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":29,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:19.915492838Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Writing ledger to /pool/int/968bbfe8-8122-4a85-96dd-954188a2f89d/config/.rss-service-plan-v3.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:19.915526369Z","hostname":"buskin","pid":1555,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/e8396e20-0493-4030-b55b-5c9e59a237cf/config/.rss-service-plan-v3.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:19.915886707Z","hostname":"buskin","pid":1555,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Service plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:19.916123593Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/plan/service.rs:812"} {"msg":"attempting to set up sled's storage: OmicronPhysicalDisksConfig { disks: [OmicronPhysicalDiskConfig { id: 22e4a77d-04b1-4a98-9cd3-77e4a13b5417, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, pool_id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool) }, OmicronPhysicalDiskConfig { id: 465144cc-1006-43a9-9104-21aebaa77e1d, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, pool_id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool) }, OmicronPhysicalDiskConfig { id: 784f2ac1-5bcf-49b9-813b-5b0df1665fe3, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, pool_id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool) }, OmicronPhysicalDiskConfig { id: dbe2cbbf-438d-44ab-8671-7f79d09f94c8, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, pool_id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool) }, OmicronPhysicalDiskConfig { id: 347eb091-9cba-4ec0-b3da-e06e8da6ebb9, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, pool_id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool) }, OmicronPhysicalDiskConfig { id: c0ca99c6-c8e0-4e93-834a-6a4f24ea22c2, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, pool_id: bd06e234-5970-4365-a0e5-6cd29811daea (zpool) }, OmicronPhysicalDiskConfig { id: 6ee844cd-ae94-45c0-9afc-a67a70fe1ed3, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, pool_id: e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool) }, OmicronPhysicalDiskConfig { id: 235ce1d8-6eb0-4300-8308-a49889040b08, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, pool_id: 4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool) }, OmicronPhysicalDiskConfig { id: 195ae64e-2a19-4f6f-a900-09099475ba32, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, pool_id: cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool) }], generation: Generation(1) }","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:19.946623404Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:348"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:19.946709586Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:45546"} {"msg":"Received OmicronPhysicalDisksEnsure { config: OmicronPhysicalDisksConfig { generation: Generation(1), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, id: 22e4a77d-04b1-4a98-9cd3-77e4a13b5417, pool_id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, id: 465144cc-1006-43a9-9104-21aebaa77e1d, pool_id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, id: 784f2ac1-5bcf-49b9-813b-5b0df1665fe3, pool_id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, id: dbe2cbbf-438d-44ab-8671-7f79d09f94c8, pool_id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, id: 347eb091-9cba-4ec0-b3da-e06e8da6ebb9, pool_id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, id: c0ca99c6-c8e0-4e93-834a-6a4f24ea22c2, pool_id: bd06e234-5970-4365-a0e5-6cd29811daea (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, id: 6ee844cd-ae94-45c0-9afc-a67a70fe1ed3, pool_id: e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, id: 235ce1d8-6eb0-4300-8308-a49889040b08, pool_id: 4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, id: 195ae64e-2a19-4f6f-a900-09099475ba32, pool_id: cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool) }] }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:19.946913701Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/968bbfe8-8122-4a85-96dd-954188a2f89d/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:19.946921981Z","hostname":"buskin","pid":1555,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/e8396e20-0493-4030-b55b-5c9e59a237cf/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:19.946936012Z","hostname":"buskin","pid":1555,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No previously-stored 'requested disks', creating new ledger","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:19.946941372Z","hostname":"buskin","pid":1555,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:681"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:19.946971912Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:339"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:19.946977233Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:19.946982013Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:19.946985963Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:34:20.040861753Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:390","component":"producer-server","component":"MetricsManager","sled_id":"8ead6dd9-9765-4776-9768-713d8773d37f","component":"SledAgent","file":"oximeter/producer/src/lib.rs:390","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"697.504336ms"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:20.047985989Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:20.047994669Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool)"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/8ead6dd9-9765-4776-9768-713d8773d37f): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-06-26T05:34:20.680423605Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"8ead6dd9-9765-4776-9768-713d8773d37f","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:34:20.881098167Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:390","component":"producer-server","component":"MetricsManager","sled_id":"8ead6dd9-9765-4776-9768-713d8773d37f","component":"SledAgent","file":"oximeter/producer/src/lib.rs:390","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"938.864015ms"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:21.074420538Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:21.261242217Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:21.261357609Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:21.26136696Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:21.26138767Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:21.26139379Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:21.261553174Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:21.261559974Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:21.799576537Z","hostname":"buskin","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:21.986857156Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/8ead6dd9-9765-4776-9768-713d8773d37f): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:34:22.185092031Z","hostname":"buskin","pid":1555,"component":"NexusNotifierTask","sled_id":"8ead6dd9-9765-4776-9768-713d8773d37f","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/8ead6dd9-9765-4776-9768-713d8773d37f): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-06-26T05:34:22.185156203Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"8ead6dd9-9765-4776-9768-713d8773d37f","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:34:22.185165723Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:390","component":"producer-server","component":"MetricsManager","sled_id":"8ead6dd9-9765-4776-9768-713d8773d37f","component":"SledAgent","file":"oximeter/producer/src/lib.rs:390","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"2.858186308s"} {"msg":"Automatically destroying dataset: oxp_2a12c7dc-fee7-4f85-ab9e-afd52f78b31f/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:22.42204859Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 38.130511ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:34:22.603530874Z","hostname":"buskin","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 98.09022ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:34:23.148581441Z","hostname":"buskin","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/8ead6dd9-9765-4776-9768-713d8773d37f): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-06-26T05:34:23.560661096Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"8ead6dd9-9765-4776-9768-713d8773d37f","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 271.984743ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:34:23.765632378Z","hostname":"buskin","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 443.40884ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:34:24.448427169Z","hostname":"buskin","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/8ead6dd9-9765-4776-9768-713d8773d37f): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:34:24.451085771Z","hostname":"buskin","pid":1555,"component":"NexusNotifierTask","sled_id":"8ead6dd9-9765-4776-9768-713d8773d37f","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/8ead6dd9-9765-4776-9768-713d8773d37f): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-06-26T05:34:24.63274534Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"8ead6dd9-9765-4776-9768-713d8773d37f","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Looking for unencrypted datasets in oxp_2a12c7dc-fee7-4f85-ab9e-afd52f78b31f","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:25.008794154Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 949.562043ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:34:25.173030596Z","hostname":"buskin","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:34:25.175549845Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:390","component":"producer-server","component":"MetricsManager","sled_id":"8ead6dd9-9765-4776-9768-713d8773d37f","component":"SledAgent","file":"oximeter/producer/src/lib.rs:390","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"4.320457823s"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:25.182253261Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:25.182264371Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:25.182270771Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:25.182276571Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:25.182283822Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:25.396396307Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:25.396430878Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:26.600848129Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:26.8284598Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:26.828503291Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:26.828511961Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:26.828536232Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:26.828542462Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:26.828743707Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:26.828751437Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/8ead6dd9-9765-4776-9768-713d8773d37f): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:34:27.395870239Z","hostname":"buskin","pid":1555,"component":"NexusNotifierTask","sled_id":"8ead6dd9-9765-4776-9768-713d8773d37f","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/8ead6dd9-9765-4776-9768-713d8773d37f): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-06-26T05:34:27.39591928Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"8ead6dd9-9765-4776-9768-713d8773d37f","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:28.382514218Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/8ead6dd9-9765-4776-9768-713d8773d37f): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-06-26T05:34:28.881678155Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"8ead6dd9-9765-4776-9768-713d8773d37f","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/8ead6dd9-9765-4776-9768-713d8773d37f): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:34:28.882132046Z","hostname":"buskin","pid":1555,"component":"NexusNotifierTask","sled_id":"8ead6dd9-9765-4776-9768-713d8773d37f","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"Automatically destroying dataset: oxp_5e9c007a-1be0-4b01-8d85-c9b89975656a/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:28.882501034Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Ensuring tfportrear0_0/ll exists in zone","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:29.621979868Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2233"} {"msg":"Finished setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:29.840984598Z","hostname":"buskin","pid":1555,"component":"VmmReservoirManager","sled_id":"8ead6dd9-9765-4776-9768-713d8773d37f","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:279"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:34:29.842779029Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:390","component":"producer-server","component":"MetricsManager","sled_id":"8ead6dd9-9765-4776-9768-713d8773d37f","component":"SledAgent","file":"oximeter/producer/src/lib.rs:390","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"5.702168202s"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/8ead6dd9-9765-4776-9768-713d8773d37f): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-06-26T05:34:29.883592421Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"8ead6dd9-9765-4776-9768-713d8773d37f","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Looking for unencrypted datasets in oxp_5e9c007a-1be0-4b01-8d85-c9b89975656a","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:29.952341176Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:29.959275777Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:29.959325439Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:29.959333629Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:29.959341079Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:29.959360079Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_2.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_2.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:29.969902315Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:29.969930276Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:30.065533816Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","zpool":"ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:30.07382313Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:30.073877131Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:30.073888341Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:30.073949643Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:30.073974393Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:30.074142967Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:30.074152578Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:30.111063389Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_d4ac128c-bdc1-48a9-b5af-10262f74baf6/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:30.119484235Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_d4ac128c-bdc1-48a9-b5af-10262f74baf6","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:30.249246673Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:30.257646689Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:30.25769505Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:30.25770468Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:30.2577127Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:30.257718861Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_3.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_3.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:30.268009401Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:30.268047242Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"4f34febf-41f3-4c60-ba39-4554d702f4af (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:30.363171121Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","zpool":"ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:30.371244009Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:30.37128184Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:30.37128812Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:30.371327121Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:30.371338561Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:30.371519876Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:30.371526806Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:30.409319758Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_4f34febf-41f3-4c60-ba39-4554d702f4af/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:30.417287193Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_4f34febf-41f3-4c60-ba39-4554d702f4af","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:30.543436287Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:30.55170254Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:30.551746291Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:30.551754501Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:30.551763131Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:30.551769451Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_4.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_4.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:30.56154993Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:30.56157872Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:30.659265429Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","zpool":"ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:30.66785466Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:30.667898451Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:30.667906951Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:30.667966582Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:30.667989183Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:30.668190798Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:30.668201578Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:30.70474832Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:30.712783148Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:30.838308027Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:30.846114019Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:30.84616877Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:30.84617506Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:30.84618212Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:30.846188371Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_5.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_5.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:30.855672032Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:30.855709023Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"bd06e234-5970-4365-a0e5-6cd29811daea (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:30.948806025Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","zpool":"ZpoolName { id: bd06e234-5970-4365-a0e5-6cd29811daea (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:30.956646978Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:30.956697279Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:30.956705859Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:30.95671663Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:30.95672288Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:30.956863883Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:30.956872823Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/8ead6dd9-9765-4776-9768-713d8773d37f): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-06-26T05:34:30.960856996Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"8ead6dd9-9765-4776-9768-713d8773d37f","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/8ead6dd9-9765-4776-9768-713d8773d37f): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:34:30.961120222Z","hostname":"buskin","pid":1555,"component":"NexusNotifierTask","sled_id":"8ead6dd9-9765-4776-9768-713d8773d37f","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:30.994347757Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_bd06e234-5970-4365-a0e5-6cd29811daea/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:31.002486587Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_bd06e234-5970-4365-a0e5-6cd29811daea","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:31.130526755Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:31.139156456Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:31.139194107Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:31.139199717Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:31.139207947Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:31.139212568Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_6.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_6.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:31.14917944Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:31.149205321Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:31.24518214Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","zpool":"ZpoolName { id: e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:31.253306639Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:31.253390841Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:31.253398952Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:31.253435442Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:31.253462903Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:31.253583276Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:31.253590936Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:31.289889463Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_e1e381d6-18e9-425e-b2e0-7789234ef986/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:31.297823098Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_e1e381d6-18e9-425e-b2e0-7789234ef986","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:31.420857019Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:31.428921307Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:31.428956448Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:31.428973348Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:31.428982058Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:31.429021839Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_7.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_7.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:31.439528375Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:31.439570045Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:31.535833082Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","zpool":"ZpoolName { id: 4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:31.545123438Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:31.545167519Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:31.545175929Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:31.545223081Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:31.545232731Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:31.545378084Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:31.545386534Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:31.58290501Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_4031e1c2-96e5-4572-96c7-90799cc1b09a/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:31.591232394Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_4031e1c2-96e5-4572-96c7-90799cc1b09a","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:31.716099428Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:31.724755779Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:31.724799731Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:31.724821721Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:379","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:31.724827331Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:456"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:31.724831801Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:121","path":"/scratch/u2_8.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_8.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:31.734734492Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:304"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:31.734770263Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:310","id":"cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:31.829890852Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","zpool":"ZpoolName { id: cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:31.838072303Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:31.838116854Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:31.838123715Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:31.838165616Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:31.838177166Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:31.838307299Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:31.838314209Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:31.873593832Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_cdd9c631-6644-4cda-aca6-05f7f580f308/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:31.881559658Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/8ead6dd9-9765-4776-9768-713d8773d37f): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-06-26T05:34:31.963346566Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"8ead6dd9-9765-4776-9768-713d8773d37f","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Looking for unencrypted datasets in oxp_cdd9c631-6644-4cda-aca6-05f7f580f308","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:32.003166734Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:32.011268754Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:477","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:32.011308764Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:394","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Writing ledger to /pool/int/968bbfe8-8122-4a85-96dd-954188a2f89d/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:32.011383646Z","hostname":"buskin","pid":1555,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:32.01154943Z","hostname":"buskin","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:41","disks":"AllDisks { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024 }, zpool_name: ZpoolName { id: 968bbfe8-8122-4a85-96dd-954188a2f89d (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025 }, zpool_name: ZpoolName { id: e8396e20-0493-4030-b55b-5c9e59a237cf (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026 }, zpool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027 }, zpool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028 }, zpool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029 }, zpool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030 }, zpool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031 }, zpool_name: ZpoolName { id: bd06e234-5970-4365-a0e5-6cd29811daea (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032 }, zpool_name: ZpoolName { id: e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033 }, zpool_name: ZpoolName { id: 4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034 }, zpool_name: ZpoolName { id: cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool), kind: External } }))}, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:32.011596991Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:549","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:32.011603971Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"Writing ledger to /pool/int/e8396e20-0493-4030-b55b-5c9e59a237cf/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:32.011716884Z","hostname":"buskin","pid":1555,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:32.012038121Z","hostname":"buskin","pid":1555,"uri":"/omicron-physical-disks","method":"PUT","req_id":"8afa2066-cdc6-4cac-be05-97a3e46114f7","remote_addr":"[fd00:1122:3344:101::1]:45546","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":12065273,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(2), zones: [OmicronZoneConfig { id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 9f65db5e-d949-4342-9d90-6d691adea120, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:32.042899881Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:32.043054615Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:64269"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:32.043301261Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:32.043307491Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9f65db5e-d949-4342-9d90-6d691adea120, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:32.09383009Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:32.093877971Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9f65db5e-d949-4342-9d90-6d691adea120, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:32.142836423Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:32.142877864Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:32.195792639Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"No ledger in /pool/int/968bbfe8-8122-4a85-96dd-954188a2f89d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:32.19584617Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/e8396e20-0493-4030-b55b-5c9e59a237cf/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:32.195867401Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:32.195911711Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:32.241612718Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:32.241669879Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:32.241682399Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_9f65db5e-d949-4342-9d90-6d691adea120","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:32.293709933Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_217ccd12-3c92-4bac-9f9c-eac91bbdf1b2","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:32.297661116Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_09c6b6ed-da1f-491b-a5c1-f925ed53c317","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:32.301610268Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_internal_dns_9f65db5e-d949-4342-9d90-6d691adea120","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:32.314223252Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_internal_dns_217ccd12-3c92-4bac-9f9c-eac91bbdf1b2","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:32.31842961Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_internal_dns_09c6b6ed-da1f-491b-a5c1-f925ed53c317","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:32.322559317Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/8ead6dd9-9765-4776-9768-713d8773d37f): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:34:32.959252022Z","hostname":"buskin","pid":1555,"component":"NexusNotifierTask","sled_id":"8ead6dd9-9765-4776-9768-713d8773d37f","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/8ead6dd9-9765-4776-9768-713d8773d37f): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-06-26T05:34:32.965023036Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"8ead6dd9-9765-4776-9768-713d8773d37f","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Profile for oxz_internal_dns_9f65db5e-d949-4342-9d90-6d691adea120:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:33.861915233Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:33.861970834Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:2::, len: 64 }"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:33.862119998Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_9f65db5e-d949-4342-9d90-6d691adea120","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_internal_dns_217ccd12-3c92-4bac-9f9c-eac91bbdf1b2:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:34.348731241Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_internal_dns_09c6b6ed-da1f-491b-a5c1-f925ed53c317:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:34.827767478Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:34.827825159Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:3::, len: 64 }"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:34.82783761Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:87","prefix":"Ipv6Prefix { addr: fd00:1122:3344:1::, len: 64 }"} {"msg":"wait for service Some(\"oxz_internal_dns_9f65db5e-d949-4342-9d90-6d691adea120\") failed: Property not found. retry in 46.509872ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:34:34.835880527Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_9f65db5e-d949-4342-9d90-6d691adea120","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:34.835920688Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_217ccd12-3c92-4bac-9f9c-eac91bbdf1b2","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:34.840222869Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_09c6b6ed-da1f-491b-a5c1-f925ed53c317","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_internal_dns_9f65db5e-d949-4342-9d90-6d691adea120\") failed: Property not found. retry in 76.415818ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:34:34.89172441Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_9f65db5e-d949-4342-9d90-6d691adea120","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_09c6b6ed-da1f-491b-a5c1-f925ed53c317\") failed: Property not found. retry in 42.61439ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:34:34.966897384Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_09c6b6ed-da1f-491b-a5c1-f925ed53c317","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_217ccd12-3c92-4bac-9f9c-eac91bbdf1b2\") failed: Property not found. retry in 70.523199ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:34:34.97356907Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_217ccd12-3c92-4bac-9f9c-eac91bbdf1b2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_9f65db5e-d949-4342-9d90-6d691adea120\") failed: Property not found. retry in 205.680285ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:34:34.981330611Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_9f65db5e-d949-4342-9d90-6d691adea120","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_09c6b6ed-da1f-491b-a5c1-f925ed53c317\") failed: Property not found. retry in 103.929768ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:34:35.017441754Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_09c6b6ed-da1f-491b-a5c1-f925ed53c317","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_217ccd12-3c92-4bac-9f9c-eac91bbdf1b2\") failed: Property not found. retry in 95.533894ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:34:35.100129383Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_217ccd12-3c92-4bac-9f9c-eac91bbdf1b2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_09c6b6ed-da1f-491b-a5c1-f925ed53c317\") failed: Property not found. retry in 200.945738ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:34:35.146554116Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_09c6b6ed-da1f-491b-a5c1-f925ed53c317","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_9f65db5e-d949-4342-9d90-6d691adea120\") failed: Property not found. retry in 480.131612ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:34:35.222062268Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_9f65db5e-d949-4342-9d90-6d691adea120","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_217ccd12-3c92-4bac-9f9c-eac91bbdf1b2\") failed: Property not found. retry in 243.694667ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:34:35.249231442Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_217ccd12-3c92-4bac-9f9c-eac91bbdf1b2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_09c6b6ed-da1f-491b-a5c1-f925ed53c317\") failed: Property not found. retry in 216.439439ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:34:35.364952782Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_09c6b6ed-da1f-491b-a5c1-f925ed53c317","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_217ccd12-3c92-4bac-9f9c-eac91bbdf1b2\") failed: Property not found. retry in 459.133643ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:34:35.506005793Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_217ccd12-3c92-4bac-9f9c-eac91bbdf1b2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_09c6b6ed-da1f-491b-a5c1-f925ed53c317\") failed: Property not found. retry in 1.132603194s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:34:35.592707836Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_09c6b6ed-da1f-491b-a5c1-f925ed53c317","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_9f65db5e-d949-4342-9d90-6d691adea120\") failed: Property not found. retry in 973.542254ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:34:35.71121167Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_9f65db5e-d949-4342-9d90-6d691adea120","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_217ccd12-3c92-4bac-9f9c-eac91bbdf1b2\") failed: Property not found. retry in 434.806277ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:34:35.973901919Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_217ccd12-3c92-4bac-9f9c-eac91bbdf1b2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_217ccd12-3c92-4bac-9f9c-eac91bbdf1b2\") failed: Property not found. retry in 959.98328ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:34:36.420398457Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_217ccd12-3c92-4bac-9f9c-eac91bbdf1b2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_9f65db5e-d949-4342-9d90-6d691adea120\") failed: Property not found. retry in 887.579205ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:34:36.694662636Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_9f65db5e-d949-4342-9d90-6d691adea120","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_09c6b6ed-da1f-491b-a5c1-f925ed53c317\") failed: Property not found. retry in 1.024535909s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:34:36.736184345Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_09c6b6ed-da1f-491b-a5c1-f925ed53c317","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_217ccd12-3c92-4bac-9f9c-eac91bbdf1b2\") failed: Property not found. retry in 1.427096956s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:34:37.389951998Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_217ccd12-3c92-4bac-9f9c-eac91bbdf1b2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Ensuring bootstrap address fdb0:18c0:4d0d:9fb2::2 exists in switch zone","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:37.545429865Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2250"} {"msg":"Adding bootstrap address","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:37.545438136Z","hostname":"buskin","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:626"} {"msg":"wait for service Some(\"oxz_internal_dns_9f65db5e-d949-4342-9d90-6d691adea120\") failed: Property not found. retry in 1.24939748s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:34:37.59148809Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_9f65db5e-d949-4342-9d90-6d691adea120","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_09c6b6ed-da1f-491b-a5c1-f925ed53c317\") failed: Property not found. retry in 829.252079ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:34:37.768579432Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_09c6b6ed-da1f-491b-a5c1-f925ed53c317","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Forwarding bootstrap traffic via oxBootstrap0 to fe80::8:20ff:fec4:4114","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:38.095022228Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2257"} {"msg":"Zone not using gateway (even though underlay is up)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:38.100824264Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2318"} {"msg":"Setting up dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:38.220332412Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2503"} {"msg":"Setting up lldpd service","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:38.466541967Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2667"} {"msg":"wait for service Some(\"oxz_internal_dns_09c6b6ed-da1f-491b-a5c1-f925ed53c317\") failed: Property not found. retry in 550.640954ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:34:38.606482212Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_09c6b6ed-da1f-491b-a5c1-f925ed53c317","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Setting up MGS service","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:38.659816346Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2382"} {"msg":"wait for service Some(\"oxz_internal_dns_217ccd12-3c92-4bac-9f9c-eac91bbdf1b2\") failed: Property not found. retry in 1.15278085s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:34:38.826346172Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_217ccd12-3c92-4bac-9f9c-eac91bbdf1b2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_9f65db5e-d949-4342-9d90-6d691adea120\") failed: Property not found. retry in 1.475412197s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:34:38.850123746Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_9f65db5e-d949-4342-9d90-6d691adea120","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Setting up wicketd service","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:38.979033044Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2417"} {"msg":"wait for service Some(\"oxz_internal_dns_09c6b6ed-da1f-491b-a5c1-f925ed53c317\") failed: Property not found. retry in 1.260551914s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:34:39.166382505Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_09c6b6ed-da1f-491b-a5c1-f925ed53c317","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Setting up mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:39.197252886Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2717"} {"msg":"Setting up mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:39.384833022Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2744"} {"msg":"Setting up tfport service","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:39.620146793Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2611"} {"msg":"Setting up Simulated SP service","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:39.816355241Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2411"} {"msg":"Re-enabling running switch zone (new address)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:39.827230954Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3803","new":"[fd00:1122:3344:101::2, ::1]","old":"[::1]"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:39.827280856Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3819"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::2, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:39.827289086Z","hostname":"buskin","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:609"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:40.389146254Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3827"} {"msg":"configuring dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:40.395688837Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3883"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:40.447225029Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2956","zone":"oxz_internal_dns_9f65db5e-d949-4342-9d90-6d691adea120"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:40.447292341Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2956","zone":"oxz_internal_dns_217ccd12-3c92-4bac-9f9c-eac91bbdf1b2"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:40.447299031Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2956","zone":"oxz_internal_dns_09c6b6ed-da1f-491b-a5c1-f925ed53c317"} {"msg":"Writing ledger to /pool/int/968bbfe8-8122-4a85-96dd-954188a2f89d/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:40.447313251Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/e8396e20-0493-4030-b55b-5c9e59a237cf/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:40.44767668Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:40.447954296Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"52d53931-b8dc-400e-b591-a6f024f3422b","remote_addr":"[fd00:1122:3344:101::1]:64269","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":8404851,"response_code":"204"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:40.449678426Z","hostname":"buskin","pid":1555,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:549"} {"msg":"configuring lldp service","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:40.536082453Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3945"} {"msg":"failed to write DNS configuration (will retry in 103.80886ms)","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:34:40.565360016Z","hostname":"buskin","pid":1555,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:569","error_message":"Communication Error: error sending request for url (http://[fd00:1122:3344:1::1]:5353/config): error trying to connect: tcp connect error: Connection refused (os error 146)"} {"msg":"configuring wicketd","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:40.634950539Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3926","rack_subnet":"fd00:1122:3344::"} {"msg":"configuring mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:40.655906528Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3973"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:40.688444317Z","hostname":"buskin","pid":1555,"dns_config_addr":"[fd00:1122:3344:2::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:549"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:40.751911368Z","hostname":"buskin","pid":1555,"dns_config_addr":"[fd00:1122:3344:3::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:549"} {"msg":"configuring mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:40.755571683Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:4000"} {"msg":"Configured all DNS servers","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:40.816805222Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:585"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:40.816861064Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:263"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:40.861925295Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:280","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:40.863444831Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:305"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:40.863451191Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:197","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Determining physical location of our switch zone at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:40.877426577Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:369"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(3), zones: [OmicronZoneConfig { id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 9f65db5e-d949-4342-9d90-6d691adea120, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: a4d358ae-9d04-40a4-b4a5-ac8418ace88d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 707beb4d-acae-4ff1-bb7e-97992526c5fb, kind: Service { id: a4d358ae-9d04-40a4-b4a5-ac8418ace88d }, name: Name(\"ntp-a4d358ae-9d04-40a4-b4a5-ac8418ace88d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 238, 27])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:40.914811959Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:40.915021254Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:48632"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:40.91530887Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:40.915318201Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Initializing 1 Uplinks on Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:40.933323991Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:417"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9f65db5e-d949-4342-9d90-6d691adea120, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:40.93629838Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:40.93630498Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9f65db5e-d949-4342-9d90-6d691adea120, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:40.956531542Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:40.956573683Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:40.975419163Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/968bbfe8-8122-4a85-96dd-954188a2f89d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:40.975492435Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/e8396e20-0493-4030-b55b-5c9e59a237cf/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:40.975690489Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:40.975816402Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Building Port Configuration","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:40.979075248Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:656"} {"msg":"Checking dendrite uptime","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:40.979169691Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:694"} {"msg":"Dendrite online","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:40.979713983Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:697","uptime":"2"} {"msg":"Configuring default uplink on switch","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:40.979750964Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:439","config":"PortSettings {\n links: {\n \"0\": LinkSettings {\n addrs: [\n 10.151.1.174,\n ],\n params: LinkCreate {\n autoneg: false,\n fec: None,\n kr: false,\n lane: Some(\n LinkId(\n 0,\n ),\n ),\n speed: Speed40G,\n },\n },\n },\n}"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:40.987674709Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:263"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:41.05114834Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:280","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:41.05374844Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:305"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:41.053755951Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:197","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:41.085766988Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:41.085840879Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:41.08584946Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:41.08587669Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:41.085885981Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:41.085894931Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:41.085903471Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:41.085908401Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:41.085916311Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:41.085925211Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:41.085932131Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:41.085938862Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:41.085947422Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:41.085952642Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:41.085957442Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:41.085965702Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:41.085970592Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:41.085976872Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:41.085981993Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:41.085986783Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:41.085991683Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:41.085996333Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:532","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:41.086002933Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:553","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:41.111510098Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:393","port":"Port { inner: PortInner { name: \"opte0\", ip: 172.30.3.5, mac: MacAddr6([168, 64, 37, 255, 238, 27]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.3.1 }, vnic: \"vopte0\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:41.111556069Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:1190","zone_type":"ntp"} {"msg":"no nat entry found for: 10.151.1.169","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:41.112170244Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.169","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:41.112178374Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"creation of nat entry successful for: 10.151.1.169","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:41.11244355Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:41.115220025Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_ntp_a4d358ae-9d04-40a4-b4a5-ac8418ace88d","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:41.139264286Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_ntp_a4d358ae-9d04-40a4-b4a5-ac8418ace88d","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:41.158394412Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_ntp_a4d358ae-9d04-40a4-b4a5-ac8418ace88d:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:41.777831225Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:41.777953098Z","hostname":"buskin","pid":1555,"zone":"oxz_ntp_a4d358ae-9d04-40a4-b4a5-ac8418ace88d","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_ntp_a4d358ae-9d04-40a4-b4a5-ac8418ace88d\") failed: Property not found. retry in 47.538158ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:34:41.912762953Z","hostname":"buskin","pid":1555,"zone":"oxz_ntp_a4d358ae-9d04-40a4-b4a5-ac8418ace88d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_a4d358ae-9d04-40a4-b4a5-ac8418ace88d\") failed: Property not found. retry in 63.890946ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:34:41.969612579Z","hostname":"buskin","pid":1555,"zone":"oxz_ntp_a4d358ae-9d04-40a4-b4a5-ac8418ace88d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_a4d358ae-9d04-40a4-b4a5-ac8418ace88d\") failed: Property not found. retry in 276.597871ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:34:42.050576778Z","hostname":"buskin","pid":1555,"zone":"oxz_ntp_a4d358ae-9d04-40a4-b4a5-ac8418ace88d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_a4d358ae-9d04-40a4-b4a5-ac8418ace88d\") failed: Property not found. retry in 431.206411ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:34:42.336803327Z","hostname":"buskin","pid":1555,"zone":"oxz_ntp_a4d358ae-9d04-40a4-b4a5-ac8418ace88d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_a4d358ae-9d04-40a4-b4a5-ac8418ace88d\") failed: Property not found. retry in 1.190454873s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:34:42.779854224Z","hostname":"buskin","pid":1555,"zone":"oxz_ntp_a4d358ae-9d04-40a4-b4a5-ac8418ace88d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_a4d358ae-9d04-40a4-b4a5-ac8418ace88d\") failed: Property not found. retry in 915.308723ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:34:43.980089947Z","hostname":"buskin","pid":1555,"zone":"oxz_ntp_a4d358ae-9d04-40a4-b4a5-ac8418ace88d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:44.262451215Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:49939"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:44.262584738Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"b74e4b14-9b87-4785-accd-f447c5db6ac3","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:49939","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":47,"response_code":"200"} {"msg":"wait for service Some(\"oxz_ntp_a4d358ae-9d04-40a4-b4a5-ac8418ace88d\") failed: Property not found. retry in 669.171337ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:34:44.904603478Z","hostname":"buskin","pid":1555,"zone":"oxz_ntp_a4d358ae-9d04-40a4-b4a5-ac8418ace88d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:34:45.550311293Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:390","component":"producer-server","component":"MetricsManager","sled_id":"8ead6dd9-9765-4776-9768-713d8773d37f","component":"SledAgent","file":"oximeter/producer/src/lib.rs:390","error":"\"no record found for Query { name: Name(\\\"_nexus._tcp.control-plane.oxide.internal.\\\"), query_type: SRV, query_class: IN }\"","delay":"22.693394619s"} {"msg":"wait for service Some(\"oxz_ntp_a4d358ae-9d04-40a4-b4a5-ac8418ace88d\") failed: Property not found. retry in 590.471286ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:34:45.58315709Z","hostname":"buskin","pid":1555,"zone":"oxz_ntp_a4d358ae-9d04-40a4-b4a5-ac8418ace88d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_a4d358ae-9d04-40a4-b4a5-ac8418ace88d\") failed: Property not found. retry in 842.577559ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:34:46.183395894Z","hostname":"buskin","pid":1555,"zone":"oxz_ntp_a4d358ae-9d04-40a4-b4a5-ac8418ace88d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:47.04772453Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2956","zone":"oxz_ntp_a4d358ae-9d04-40a4-b4a5-ac8418ace88d"} {"msg":"Writing ledger to /pool/int/968bbfe8-8122-4a85-96dd-954188a2f89d/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:47.047798252Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/e8396e20-0493-4030-b55b-5c9e59a237cf/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:47.048124929Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:47.048444507Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"77778742-78f9-4856-9416-fd261faca2e6","remote_addr":"[fd00:1122:3344:101::1]:48632","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":6133356,"response_code":"204"} {"msg":"Waiting for rack time synchronization","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:47.050225678Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:626"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:47.079158554Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:47.079303677Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:49092"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_a4d358ae-9d04-40a4-b4a5-ac8418ace88d': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:47.088974612Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3533"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:47.089102646Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"f8fe883e-568b-4aee-ae0a-55b5598c9f9f","remote_addr":"[fd00:1122:3344:101::1]:49092","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":9625,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:34:47.091328497Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:47.216905497Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:47.216988519Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:36566"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_a4d358ae-9d04-40a4-b4a5-ac8418ace88d': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:51.225992947Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3533"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:51.226109219Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"9c3e5198-36eb-4319-b80c-4230d1f1fac8","remote_addr":"[fd00:1122:3344:101::1]:36566","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":4008999,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:34:51.229809176Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:51.380574493Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:51.380704176Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:42590"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_a4d358ae-9d04-40a4-b4a5-ac8418ace88d': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:51.389614714Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3533"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:51.389732577Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"12b6dfcb-7ae1-4e34-b6a8-6364db2f11c0","remote_addr":"[fd00:1122:3344:101::1]:42590","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":8867,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:34:51.39243934Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:51.617722426Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:51.617820098Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:47104"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_a4d358ae-9d04-40a4-b4a5-ac8418ace88d': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:51.626004629Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3533"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:51.626094312Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"09e0bc62-ba60-4343-a944-78a90df173f9","remote_addr":"[fd00:1122:3344:101::1]:47104","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":8159,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:34:51.627808662Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:51.805666071Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:51.805783414Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:37289"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:55.814752671Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"3bc136b3-2392-4a00-9eaa-d43cc3181a48","remote_addr":"[fd00:1122:3344:101::1]:37289","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":4008871,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:55.817968015Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:34:55.817976806Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:56.135734409Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:56.135873523Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:52100"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:56.144529455Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"eaaf198b-e6f7-4fc6-a913-e5c809f5769c","remote_addr":"[fd00:1122:3344:101::1]:52100","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":8544,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:56.147626517Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:34:56.147640647Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:56.419433159Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:56.419539361Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:39467"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:56.427573359Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"654d3cb6-9236-47bf-b400-ac7e864921b7","remote_addr":"[fd00:1122:3344:101::1]:39467","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":7901,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:56.431520261Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:34:56.431525811Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:56.770525451Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:56.770631083Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:57978"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:56.777743549Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"a10f9d78-dfc6-42c7-b342-d56138687113","remote_addr":"[fd00:1122:3344:101::1]:57978","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":7012,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:56.779754856Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:34:56.779761306Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:57.072310722Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:57.072473105Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:55011"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:57.081423644Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"06d1e42c-2630-4935-9421-846889dabdd2","remote_addr":"[fd00:1122:3344:101::1]:55011","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":8895,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:57.085472329Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:34:57.085478639Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:57.601237313Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:57.601376746Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:39126"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:57.61097206Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"ff63a862-3745-46a8-8fcc-3efa4d60fece","remote_addr":"[fd00:1122:3344:101::1]:39126","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":9564,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:57.61442411Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:34:57.614433481Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:58.258771624Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:58.258887547Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:46192"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:58.267452937Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"ed60ce47-6316-4692-acc9-e701745e775d","remote_addr":"[fd00:1122:3344:101::1]:46192","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":8459,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:58.269575116Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:34:58.269582016Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:656","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:59.360726187Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:606"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:59.360898391Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:53586"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:59.383037787Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"6dd837c2-cec4-4e3a-8cd9-ba71f3c5c064","remote_addr":"[fd00:1122:3344:101::1]:53586","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":22065,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: true, ref_id: 395126907, ip_addr: 23.141.40.123, stratum: 3, ref_time: 1719380098.4733157, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:59.386980919Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:634"} {"msg":"Finished setting up Internal DNS and NTP","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:59.386987009Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1224"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 9f65db5e-d949-4342-9d90-6d691adea120, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: a4d358ae-9d04-40a4-b4a5-ac8418ace88d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 707beb4d-acae-4ff1-bb7e-97992526c5fb, kind: Service { id: a4d358ae-9d04-40a4-b4a5-ac8418ace88d }, name: Name(\"ntp-a4d358ae-9d04-40a4-b4a5-ac8418ace88d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 238, 27])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e3552cf6-27c7-4100-942b-c5877c673046, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } } } }, OmicronZoneConfig { id: 9a0125d1-b636-4039-9e54-dea22726a38a, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } } } }, OmicronZoneConfig { id: 7c851202-bc92-4a32-b946-68379ad242d7, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 653e8bac-3f5b-49fe-a9f9-e9f0546059e9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External } } } }, OmicronZoneConfig { id: e4ffe9aa-395e-4536-a4d0-5cf0bc814584, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:59.416180381Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:59.416321574Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:56114"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:59.416622041Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:59.416631241Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9f65db5e-d949-4342-9d90-6d691adea120, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:59.437857306Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:59.437908897Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9f65db5e-d949-4342-9d90-6d691adea120, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:59.457657378Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:59.457705819Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e3552cf6-27c7-4100-942b-c5877c673046, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:59.47829831Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:59.478346241Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e3552cf6-27c7-4100-942b-c5877c673046, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a0125d1-b636-4039-9e54-dea22726a38a, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:59.536437376Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:34:59.536483177Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a0125d1-b636-4039-9e54-dea22726a38a, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c851202-bc92-4a32-b946-68379ad242d7, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:00.035807286Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:00.035862837Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7c851202-bc92-4a32-b946-68379ad242d7, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 653e8bac-3f5b-49fe-a9f9-e9f0546059e9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:00.091909985Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:00.091958626Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 653e8bac-3f5b-49fe-a9f9-e9f0546059e9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e4ffe9aa-395e-4536-a4d0-5cf0bc814584, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:00.14657641Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:00.146624831Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e4ffe9aa-395e-4536-a4d0-5cf0bc814584, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:00.20013125Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/968bbfe8-8122-4a85-96dd-954188a2f89d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:00.200202501Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/e8396e20-0493-4030-b55b-5c9e59a237cf/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:00.200427557Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:00.20057683Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:00.286943445Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:00.286997846Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:00.287004167Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:00.287009797Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:00.287015317Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_e3552cf6-27c7-4100-942b-c5877c673046","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:00.384217415Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_7c851202-bc92-4a32-b946-68379ad242d7","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:00.388158137Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_9a0125d1-b636-4039-9e54-dea22726a38a","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:00.392267853Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_653e8bac-3f5b-49fe-a9f9-e9f0546059e9","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:00.396889901Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_e4ffe9aa-395e-4536-a4d0-5cf0bc814584","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:00.401712253Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_cockroachdb_e3552cf6-27c7-4100-942b-c5877c673046","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:00.406205318Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_7c851202-bc92-4a32-b946-68379ad242d7","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:00.410247992Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_9a0125d1-b636-4039-9e54-dea22726a38a","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:00.415539666Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_653e8bac-3f5b-49fe-a9f9-e9f0546059e9","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:00.420922521Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_e4ffe9aa-395e-4536-a4d0-5cf0bc814584","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:00.425683402Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_cockroachdb_9a0125d1-b636-4039-9e54-dea22726a38a:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:02.784227045Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_cockroachdb_653e8bac-3f5b-49fe-a9f9-e9f0546059e9:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:02.784419899Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:02.784427419Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_9a0125d1-b636-4039-9e54-dea22726a38a","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:02.790532382Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_653e8bac-3f5b-49fe-a9f9-e9f0546059e9","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_cockroachdb_e3552cf6-27c7-4100-942b-c5877c673046:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:02.795773324Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:02.795883157Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_e3552cf6-27c7-4100-942b-c5877c673046","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_cockroachdb_7c851202-bc92-4a32-b946-68379ad242d7:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:02.869258909Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:02.869374471Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_7c851202-bc92-4a32-b946-68379ad242d7","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_cockroachdb_e4ffe9aa-395e-4536-a4d0-5cf0bc814584:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:02.894091988Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:02.894216891Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_e4ffe9aa-395e-4536-a4d0-5cf0bc814584","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_cockroachdb_9a0125d1-b636-4039-9e54-dea22726a38a\") failed: Property not found. retry in 64.383884ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:02.932714569Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_9a0125d1-b636-4039-9e54-dea22726a38a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_653e8bac-3f5b-49fe-a9f9-e9f0546059e9\") failed: Property not found. retry in 49.406409ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:02.94046156Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_653e8bac-3f5b-49fe-a9f9-e9f0546059e9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e3552cf6-27c7-4100-942b-c5877c673046\") failed: Property not found. retry in 42.236953ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:02.947590216Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_e3552cf6-27c7-4100-942b-c5877c673046","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e3552cf6-27c7-4100-942b-c5877c673046\") failed: Property not found. retry in 143.179462ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:03.060260514Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_e3552cf6-27c7-4100-942b-c5877c673046","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_7c851202-bc92-4a32-b946-68379ad242d7\") failed: Property not found. retry in 56.565677ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:03.34255154Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_7c851202-bc92-4a32-b946-68379ad242d7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9a0125d1-b636-4039-9e54-dea22726a38a\") failed: Property not found. retry in 143.571645ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:03.584095325Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_9a0125d1-b636-4039-9e54-dea22726a38a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_653e8bac-3f5b-49fe-a9f9-e9f0546059e9\") failed: Property not found. retry in 112.613227ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:03.920811181Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_653e8bac-3f5b-49fe-a9f9-e9f0546059e9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e4ffe9aa-395e-4536-a4d0-5cf0bc814584\") failed: Property not found. retry in 47.198615ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:04.096384937Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_e4ffe9aa-395e-4536-a4d0-5cf0bc814584","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e3552cf6-27c7-4100-942b-c5877c673046\") failed: Property not found. retry in 241.131467ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:04.77506017Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_e3552cf6-27c7-4100-942b-c5877c673046","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_7c851202-bc92-4a32-b946-68379ad242d7\") failed: Property not found. retry in 91.145827ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:04.861944117Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_7c851202-bc92-4a32-b946-68379ad242d7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9a0125d1-b636-4039-9e54-dea22726a38a\") failed: Property not found. retry in 179.49591ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:05.061988694Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_9a0125d1-b636-4039-9e54-dea22726a38a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_653e8bac-3f5b-49fe-a9f9-e9f0546059e9\") failed: Property not found. retry in 109.092953ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:05.200995087Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_653e8bac-3f5b-49fe-a9f9-e9f0546059e9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e4ffe9aa-395e-4536-a4d0-5cf0bc814584\") failed: Property not found. retry in 142.014323ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:05.270887058Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_e4ffe9aa-395e-4536-a4d0-5cf0bc814584","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e3552cf6-27c7-4100-942b-c5877c673046\") failed: Property not found. retry in 227.443397ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:05.310156474Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_e3552cf6-27c7-4100-942b-c5877c673046","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_7c851202-bc92-4a32-b946-68379ad242d7\") failed: Property not found. retry in 130.001303ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:05.350502435Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_7c851202-bc92-4a32-b946-68379ad242d7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9a0125d1-b636-4039-9e54-dea22726a38a\") failed: Property not found. retry in 284.970008ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:05.432280283Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_9a0125d1-b636-4039-9e54-dea22726a38a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_653e8bac-3f5b-49fe-a9f9-e9f0546059e9\") failed: Property not found. retry in 210.619461ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:05.456819625Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_653e8bac-3f5b-49fe-a9f9-e9f0546059e9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e4ffe9aa-395e-4536-a4d0-5cf0bc814584\") failed: Property not found. retry in 256.11728ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:05.476224588Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_e4ffe9aa-395e-4536-a4d0-5cf0bc814584","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_7c851202-bc92-4a32-b946-68379ad242d7\") failed: Property not found. retry in 452.182414ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:05.499474361Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_7c851202-bc92-4a32-b946-68379ad242d7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e3552cf6-27c7-4100-942b-c5877c673046\") failed: Property not found. retry in 611.617689ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:05.552212211Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_e3552cf6-27c7-4100-942b-c5877c673046","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_653e8bac-3f5b-49fe-a9f9-e9f0546059e9\") failed: Property not found. retry in 669.549177ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:05.679103611Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_653e8bac-3f5b-49fe-a9f9-e9f0546059e9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9a0125d1-b636-4039-9e54-dea22726a38a\") failed: Property not found. retry in 1.020388012s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:05.69317439Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_9a0125d1-b636-4039-9e54-dea22726a38a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e4ffe9aa-395e-4536-a4d0-5cf0bc814584\") failed: Property not found. retry in 553.545105ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:05.743638157Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_e4ffe9aa-395e-4536-a4d0-5cf0bc814584","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_7c851202-bc92-4a32-b946-68379ad242d7\") failed: Property not found. retry in 443.083823ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:05.961589652Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_7c851202-bc92-4a32-b946-68379ad242d7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e3552cf6-27c7-4100-942b-c5877c673046\") failed: Property not found. retry in 699.744869ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:06.175602765Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_e3552cf6-27c7-4100-942b-c5877c673046","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e4ffe9aa-395e-4536-a4d0-5cf0bc814584\") failed: Property not found. retry in 1.045642594s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:06.308000448Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_e4ffe9aa-395e-4536-a4d0-5cf0bc814584","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_653e8bac-3f5b-49fe-a9f9-e9f0546059e9\") failed: Property not found. retry in 646.646873ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:06.358806241Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_653e8bac-3f5b-49fe-a9f9-e9f0546059e9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_7c851202-bc92-4a32-b946-68379ad242d7\") failed: Property not found. retry in 986.877217ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:06.414063917Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_7c851202-bc92-4a32-b946-68379ad242d7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9a0125d1-b636-4039-9e54-dea22726a38a\") failed: Property not found. retry in 1.298855555s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:06.723046675Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_9a0125d1-b636-4039-9e54-dea22726a38a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e3552cf6-27c7-4100-942b-c5877c673046\") failed: Property not found. retry in 1.435181862s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:06.884196865Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_e3552cf6-27c7-4100-942b-c5877c673046","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_653e8bac-3f5b-49fe-a9f9-e9f0546059e9\") failed: Property not found. retry in 1.030562564s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:07.014086005Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_653e8bac-3f5b-49fe-a9f9-e9f0546059e9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e4ffe9aa-395e-4536-a4d0-5cf0bc814584\") failed: Property not found. retry in 1.003753553s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:07.363018715Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_e4ffe9aa-395e-4536-a4d0-5cf0bc814584","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_7c851202-bc92-4a32-b946-68379ad242d7\") failed: Property not found. retry in 801.605676ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:07.410000382Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_7c851202-bc92-4a32-b946-68379ad242d7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9a0125d1-b636-4039-9e54-dea22726a38a\") failed: Property not found. retry in 688.725668ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:08.034543082Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_9a0125d1-b636-4039-9e54-dea22726a38a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_653e8bac-3f5b-49fe-a9f9-e9f0546059e9\") failed: Property not found. retry in 1.250953253s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:08.05501222Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_653e8bac-3f5b-49fe-a9f9-e9f0546059e9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_7c851202-bc92-4a32-b946-68379ad242d7\") failed: Property not found. retry in 1.322038492s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:08.22389281Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_7c851202-bc92-4a32-b946-68379ad242d7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e3552cf6-27c7-4100-942b-c5877c673046\") failed: Property not found. retry in 1.452297185s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:08.33223404Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_e3552cf6-27c7-4100-942b-c5877c673046","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e4ffe9aa-395e-4536-a4d0-5cf0bc814584\") failed: Property not found. retry in 758.208511ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:08.381038255Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_e4ffe9aa-395e-4536-a4d0-5cf0bc814584","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9a0125d1-b636-4039-9e54-dea22726a38a\") failed: Property not found. retry in 803.260295ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:08.735126336Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_9a0125d1-b636-4039-9e54-dea22726a38a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e4ffe9aa-395e-4536-a4d0-5cf0bc814584\") failed: Property not found. retry in 1.445821248s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:09.150112227Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_e4ffe9aa-395e-4536-a4d0-5cf0bc814584","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_653e8bac-3f5b-49fe-a9f9-e9f0546059e9\") failed: Property not found. retry in 795.400458ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:09.314743568Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_653e8bac-3f5b-49fe-a9f9-e9f0546059e9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9a0125d1-b636-4039-9e54-dea22726a38a\") failed: Property not found. retry in 1.265204561s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:09.546803312Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_9a0125d1-b636-4039-9e54-dea22726a38a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_7c851202-bc92-4a32-b946-68379ad242d7\") failed: Property not found. retry in 581.292401ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:09.555396632Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_7c851202-bc92-4a32-b946-68379ad242d7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e3552cf6-27c7-4100-942b-c5877c673046\") failed: Property not found. retry in 523.853773ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:09.792671558Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_e3552cf6-27c7-4100-942b-c5877c673046","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_653e8bac-3f5b-49fe-a9f9-e9f0546059e9\") failed: Property not found. retry in 706.611734ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:10.119654216Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_653e8bac-3f5b-49fe-a9f9-e9f0546059e9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_7c851202-bc92-4a32-b946-68379ad242d7\") failed: Property not found. retry in 565.008675ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:10.144919326Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_7c851202-bc92-4a32-b946-68379ad242d7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e3552cf6-27c7-4100-942b-c5877c673046\") failed: Property not found. retry in 996.684812ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:10.32469875Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_e3552cf6-27c7-4100-942b-c5877c673046","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e4ffe9aa-395e-4536-a4d0-5cf0bc814584\") failed: Property not found. retry in 857.066442ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:10.606211977Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_e4ffe9aa-395e-4536-a4d0-5cf0bc814584","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_7c851202-bc92-4a32-b946-68379ad242d7\") failed: Property not found. retry in 907.044952ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:10.721012443Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_7c851202-bc92-4a32-b946-68379ad242d7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9a0125d1-b636-4039-9e54-dea22726a38a\") failed: Property not found. retry in 900.838641ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:10.820397604Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_9a0125d1-b636-4039-9e54-dea22726a38a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_653e8bac-3f5b-49fe-a9f9-e9f0546059e9\") failed: Property not found. retry in 958.218917ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:10.835062846Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_653e8bac-3f5b-49fe-a9f9-e9f0546059e9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e3552cf6-27c7-4100-942b-c5877c673046\") failed: Property not found. retry in 1.028658845s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:11.331634359Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_e3552cf6-27c7-4100-942b-c5877c673046","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e4ffe9aa-395e-4536-a4d0-5cf0bc814584\") failed: Property not found. retry in 1.002634976s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:11.473602923Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_e4ffe9aa-395e-4536-a4d0-5cf0bc814584","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_7c851202-bc92-4a32-b946-68379ad242d7\") failed: Property not found. retry in 868.264354ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:11.639474983Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_7c851202-bc92-4a32-b946-68379ad242d7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9a0125d1-b636-4039-9e54-dea22726a38a\") failed: Property not found. retry in 1.227577697s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:11.73192686Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_9a0125d1-b636-4039-9e54-dea22726a38a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_653e8bac-3f5b-49fe-a9f9-e9f0546059e9\") failed: Property not found. retry in 1.237763676s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:11.807908203Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_653e8bac-3f5b-49fe-a9f9-e9f0546059e9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e3552cf6-27c7-4100-942b-c5877c673046\") failed: Property not found. retry in 1.179374749s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:12.37063152Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_e3552cf6-27c7-4100-942b-c5877c673046","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e4ffe9aa-395e-4536-a4d0-5cf0bc814584\") failed: Property not found. retry in 864.856142ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:12.485743085Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_e4ffe9aa-395e-4536-a4d0-5cf0bc814584","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9a0125d1-b636-4039-9e54-dea22726a38a\") failed: Property not found. retry in 1.286838436s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:12.968967809Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_9a0125d1-b636-4039-9e54-dea22726a38a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_653e8bac-3f5b-49fe-a9f9-e9f0546059e9\") failed: Property not found. retry in 1.252326757s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:13.054778961Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_653e8bac-3f5b-49fe-a9f9-e9f0546059e9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e4ffe9aa-395e-4536-a4d0-5cf0bc814584\") failed: Property not found. retry in 1.365189711s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:13.360333539Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_e4ffe9aa-395e-4536-a4d0-5cf0bc814584","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e3552cf6-27c7-4100-942b-c5877c673046\") failed: Property not found. retry in 1.078344009s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:13.558197555Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_e3552cf6-27c7-4100-942b-c5877c673046","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9a0125d1-b636-4039-9e54-dea22726a38a\") failed: Property not found. retry in 611.209933ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:14.264328329Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_9a0125d1-b636-4039-9e54-dea22726a38a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_653e8bac-3f5b-49fe-a9f9-e9f0546059e9\") failed: Property not found. retry in 1.363869397s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:14.316814694Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_653e8bac-3f5b-49fe-a9f9-e9f0546059e9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e3552cf6-27c7-4100-942b-c5877c673046\") failed: Property not found. retry in 1.14209853s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:14.645924742Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_e3552cf6-27c7-4100-942b-c5877c673046","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:14.653633361Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:49087"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:14.653842066Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"5a5841dd-d09d-4261-94e0-fd0011beb569","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:49087","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":109,"response_code":"200"} {"msg":"wait for service Some(\"oxz_cockroachdb_e4ffe9aa-395e-4536-a4d0-5cf0bc814584\") failed: Property not found. retry in 1.25397777s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:14.735508292Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_e4ffe9aa-395e-4536-a4d0-5cf0bc814584","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9a0125d1-b636-4039-9e54-dea22726a38a\") failed: Property not found. retry in 1.234598812s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:14.884841295Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_9a0125d1-b636-4039-9e54-dea22726a38a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_653e8bac-3f5b-49fe-a9f9-e9f0546059e9\") failed: Property not found. retry in 587.506681ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:15.691590656Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_653e8bac-3f5b-49fe-a9f9-e9f0546059e9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e3552cf6-27c7-4100-942b-c5877c673046\") failed: Property not found. retry in 1.106226672s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:15.797083847Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_e3552cf6-27c7-4100-942b-c5877c673046","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e4ffe9aa-395e-4536-a4d0-5cf0bc814584\") failed: Property not found. retry in 1.21044917s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:15.999778621Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_e4ffe9aa-395e-4536-a4d0-5cf0bc814584","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9a0125d1-b636-4039-9e54-dea22726a38a\") failed: Property not found. retry in 605.672253ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:16.127811313Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_9a0125d1-b636-4039-9e54-dea22726a38a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_653e8bac-3f5b-49fe-a9f9-e9f0546059e9\") failed: Property not found. retry in 588.187308ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:16.287785365Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_653e8bac-3f5b-49fe-a9f9-e9f0546059e9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9a0125d1-b636-4039-9e54-dea22726a38a\") failed: Property not found. retry in 1.372520442s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:16.741853773Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_9a0125d1-b636-4039-9e54-dea22726a38a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_653e8bac-3f5b-49fe-a9f9-e9f0546059e9\") failed: Property not found. retry in 681.714852ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:16.884740877Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_653e8bac-3f5b-49fe-a9f9-e9f0546059e9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e3552cf6-27c7-4100-942b-c5877c673046\") failed: Property not found. retry in 1.028482381s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:16.911718051Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_e3552cf6-27c7-4100-942b-c5877c673046","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e4ffe9aa-395e-4536-a4d0-5cf0bc814584\") failed: Property not found. retry in 1.113700721s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:17.218774715Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_e4ffe9aa-395e-4536-a4d0-5cf0bc814584","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_653e8bac-3f5b-49fe-a9f9-e9f0546059e9\") failed: Property not found. retry in 889.861834ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:17.575203699Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_653e8bac-3f5b-49fe-a9f9-e9f0546059e9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e3552cf6-27c7-4100-942b-c5877c673046\") failed: Property not found. retry in 1.034503032s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:17.949247415Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_e3552cf6-27c7-4100-942b-c5877c673046","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9a0125d1-b636-4039-9e54-dea22726a38a\") failed: Property not found. retry in 800.988074ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:18.124257108Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_9a0125d1-b636-4039-9e54-dea22726a38a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e4ffe9aa-395e-4536-a4d0-5cf0bc814584\") failed: Property not found. retry in 919.783042ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:18.341346393Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_e4ffe9aa-395e-4536-a4d0-5cf0bc814584","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_653e8bac-3f5b-49fe-a9f9-e9f0546059e9\") failed: Property not found. retry in 1.254158132s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:18.474336406Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_653e8bac-3f5b-49fe-a9f9-e9f0546059e9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9a0125d1-b636-4039-9e54-dea22726a38a\") failed: Property not found. retry in 1.262280431s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:18.934926301Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_9a0125d1-b636-4039-9e54-dea22726a38a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e3552cf6-27c7-4100-942b-c5877c673046\") failed: Property not found. retry in 1.260846996s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:18.991999743Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_e3552cf6-27c7-4100-942b-c5877c673046","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e4ffe9aa-395e-4536-a4d0-5cf0bc814584\") failed: Property not found. retry in 579.215125ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:19.269591379Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_e4ffe9aa-395e-4536-a4d0-5cf0bc814584","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_653e8bac-3f5b-49fe-a9f9-e9f0546059e9\") failed: Property not found. retry in 815.323358ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:19.737981626Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_653e8bac-3f5b-49fe-a9f9-e9f0546059e9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e4ffe9aa-395e-4536-a4d0-5cf0bc814584\") failed: Property not found. retry in 857.19045ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:19.857022173Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_e4ffe9aa-395e-4536-a4d0-5cf0bc814584","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9a0125d1-b636-4039-9e54-dea22726a38a\") failed: Property not found. retry in 640.570407ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:20.206684101Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_9a0125d1-b636-4039-9e54-dea22726a38a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e3552cf6-27c7-4100-942b-c5877c673046\") failed: Property not found. retry in 771.659094ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:20.26195655Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_e3552cf6-27c7-4100-942b-c5877c673046","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_653e8bac-3f5b-49fe-a9f9-e9f0546059e9\") failed: Property not found. retry in 1.179283911s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:20.562623514Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_653e8bac-3f5b-49fe-a9f9-e9f0546059e9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e4ffe9aa-395e-4536-a4d0-5cf0bc814584\") failed: Property not found. retry in 841.277159ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:20.724183384Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_e4ffe9aa-395e-4536-a4d0-5cf0bc814584","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_9a0125d1-b636-4039-9e54-dea22726a38a\") failed: Property not found. retry in 1.128380257s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:20.856011879Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_9a0125d1-b636-4039-9e54-dea22726a38a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e3552cf6-27c7-4100-942b-c5877c673046\") failed: Property not found. retry in 698.103846ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:21.042310576Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_e3552cf6-27c7-4100-942b-c5877c673046","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e4ffe9aa-395e-4536-a4d0-5cf0bc814584\") failed: Property not found. retry in 759.171494ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:21.574221115Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_e4ffe9aa-395e-4536-a4d0-5cf0bc814584","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_e3552cf6-27c7-4100-942b-c5877c673046\") failed: Property not found. retry in 1.451730649s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:21.751018749Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_e3552cf6-27c7-4100-942b-c5877c673046","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_653e8bac-3f5b-49fe-a9f9-e9f0546059e9\") failed: Property not found. retry in 1.494234065s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:21.759589939Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_653e8bac-3f5b-49fe-a9f9-e9f0546059e9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:23.281009483Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2956","zone":"oxz_cockroachdb_e3552cf6-27c7-4100-942b-c5877c673046"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:23.281095215Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2956","zone":"oxz_cockroachdb_7c851202-bc92-4a32-b946-68379ad242d7"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:23.281104275Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2956","zone":"oxz_cockroachdb_9a0125d1-b636-4039-9e54-dea22726a38a"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:23.281110385Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2956","zone":"oxz_cockroachdb_653e8bac-3f5b-49fe-a9f9-e9f0546059e9"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:23.281115925Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2956","zone":"oxz_cockroachdb_e4ffe9aa-395e-4536-a4d0-5cf0bc814584"} {"msg":"Writing ledger to /pool/int/968bbfe8-8122-4a85-96dd-954188a2f89d/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:23.281167616Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/e8396e20-0493-4030-b55b-5c9e59a237cf/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:23.281561079Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:23.281918134Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"ac6f3825-709e-4998-ace0-e93850797df8","remote_addr":"[fd00:1122:3344:101::1]:56114","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":23984389,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:23.317119725Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:64386"} {"msg":"Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::6]:32221","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:23.354861036Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3367"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:23.662418871Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:423","component":"producer-server","component":"MetricsManager","sled_id":"8ead6dd9-9765-4776-9768-713d8773d37f","component":"SledAgent","file":"oximeter/producer/src/lib.rs:423","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"181.165734ms"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:38.890640705Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:423","component":"producer-server","component":"MetricsManager","sled_id":"8ead6dd9-9765-4776-9768-713d8773d37f","component":"SledAgent","file":"oximeter/producer/src/lib.rs:423","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"523.392049ms"} {"msg":"Formatting CRDB","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:42.628855276Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3385"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:44.706489477Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:46248"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:44.707720056Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"39e403ee-b859-4d44-ba1e-347349fe292b","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:46248","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":908,"response_code":"200"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:54.461867041Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:423","component":"producer-server","component":"MetricsManager","sled_id":"8ead6dd9-9765-4776-9768-713d8773d37f","component":"SledAgent","file":"oximeter/producer/src/lib.rs:423","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"661.525225ms"} {"msg":"Formatting CRDB - Completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:55.72155685Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3408"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:55.722385168Z","hostname":"buskin","pid":1555,"uri":"/cockroachdb","method":"POST","req_id":"e77f5163-93fc-4f4e-b187-f1f4a56c5aa2","remote_addr":"[fd00:1122:3344:101::1]:64386","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":32401981,"response_code":"204"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 9f65db5e-d949-4342-9d90-6d691adea120, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: a4d358ae-9d04-40a4-b4a5-ac8418ace88d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 707beb4d-acae-4ff1-bb7e-97992526c5fb, kind: Service { id: a4d358ae-9d04-40a4-b4a5-ac8418ace88d }, name: Name(\"ntp-a4d358ae-9d04-40a4-b4a5-ac8418ace88d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 238, 27])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e3552cf6-27c7-4100-942b-c5877c673046, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } } } }, OmicronZoneConfig { id: 9a0125d1-b636-4039-9e54-dea22726a38a, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } } } }, OmicronZoneConfig { id: 7c851202-bc92-4a32-b946-68379ad242d7, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 653e8bac-3f5b-49fe-a9f9-e9f0546059e9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External } } } }, OmicronZoneConfig { id: e4ffe9aa-395e-4536-a4d0-5cf0bc814584, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 48ce280c-ebea-49f3-9829-dc9b4d018a02, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: c2791662-4c87-4a63-a67a-4d6489fb4eaf, kind: Service { id: 48ce280c-ebea-49f3-9829-dc9b4d018a02 }, name: Name(\"external-dns-48ce280c-ebea-49f3-9829-dc9b4d018a02\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 86])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 38cec1c6-d097-4604-b9cb-30a5fab57e35, kind: Service { id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94 }, name: Name(\"external-dns-9f29eb53-b3d5-4fc1-b4d8-a252757e0c94\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 215])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 67179f00-d639-4f5b-b3c6-71d676113139, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: a724cf8c-c466-4981-aa80-a17ab2349f83, kind: Service { id: 67179f00-d639-4f5b-b3c6-71d676113139 }, name: Name(\"nexus-67179f00-d639-4f5b-b3c6-71d676113139\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 55])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7e2a6156-04b4-4797-8bed-d0c5c9cbe752, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: e31530f0-72e9-4544-bfbb-847c405d46c8, kind: Service { id: 7e2a6156-04b4-4797-8bed-d0c5c9cbe752 }, name: Name(\"nexus-7e2a6156-04b4-4797-8bed-d0c5c9cbe752\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 20a0479c-19b7-40a0-a935-5ff5399f7d02, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 79c95d4d-cff5-4419-8f3e-a6e216736b73, kind: Service { id: 20a0479c-19b7-40a0-a935-5ff5399f7d02 }, name: Name(\"nexus-20a0479c-19b7-40a0-a935-5ff5399f7d02\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 249])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: e42e8525-00d2-488f-a732-5c9514a5c7b4, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 96bb9301-f50f-4259-8a1b-169ec98cbff3, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } } } }, OmicronZoneConfig { id: 966bee63-3642-4e7a-a159-17eb59b72717, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: da8be60e-c115-4289-8526-a1ad7b366b93, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: d72762b4-6a43-4e1c-92b1-b432c1731075, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: c3d43ade-9569-403a-a6cc-494ec947793f, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } } } }, OmicronZoneConfig { id: f09c8d14-475d-49d8-b03f-d3a0b7b880cc, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } } } }, OmicronZoneConfig { id: b202e971-de3b-4a73-a534-ee56c23e79a3, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2b78106a-b661-444a-a40f-011ae4f0911a, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External } } } }, OmicronZoneConfig { id: f64b0db4-4e9f-4666-b267-c9abe9f8b189, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 56d79ac6-bb86-49e7-905e-343e52dbbb90, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd06e234-5970-4365-a0e5-6cd29811daea (zpool), kind: External } } } }, OmicronZoneConfig { id: 010164b7-56ba-4dda-8f67-3d5e05751e2e, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3488ba32-8748-4793-9389-5ea882fa963c, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool), kind: External } } } }, OmicronZoneConfig { id: 89e132b1-d133-478e-a8f7-cf9f30b211c9, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:55.763984406Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:55.76484181Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::1]:39162"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:55.768459564Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:55.768473125Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9f65db5e-d949-4342-9d90-6d691adea120, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:55.796169721Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:55.796217392Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9f65db5e-d949-4342-9d90-6d691adea120, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:55.815773078Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:55.815814919Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e3552cf6-27c7-4100-942b-c5877c673046, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:55.835938559Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:55.83598823Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e3552cf6-27c7-4100-942b-c5877c673046, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a0125d1-b636-4039-9e54-dea22726a38a, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:55.855476995Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:55.855518518Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a0125d1-b636-4039-9e54-dea22726a38a, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c851202-bc92-4a32-b946-68379ad242d7, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:55.87455984Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:55.874605151Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7c851202-bc92-4a32-b946-68379ad242d7, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 653e8bac-3f5b-49fe-a9f9-e9f0546059e9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:55.893644235Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:55.893688856Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 653e8bac-3f5b-49fe-a9f9-e9f0546059e9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e4ffe9aa-395e-4536-a4d0-5cf0bc814584, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:55.912251739Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:55.91229364Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e4ffe9aa-395e-4536-a4d0-5cf0bc814584, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48ce280c-ebea-49f3-9829-dc9b4d018a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:55.931222442Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:55.931271883Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 48ce280c-ebea-49f3-9829-dc9b4d018a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:56.013274356Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:56.013330537Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 96bb9301-f50f-4259-8a1b-169ec98cbff3, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:56.104367301Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:56.104432093Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 96bb9301-f50f-4259-8a1b-169ec98cbff3, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c3d43ade-9569-403a-a6cc-494ec947793f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:56.194952674Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:56.195005106Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c3d43ade-9569-403a-a6cc-494ec947793f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f09c8d14-475d-49d8-b03f-d3a0b7b880cc, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:56.288478686Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:56.288531088Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f09c8d14-475d-49d8-b03f-d3a0b7b880cc, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b202e971-de3b-4a73-a534-ee56c23e79a3, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:56.381527827Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:56.381579528Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b202e971-de3b-4a73-a534-ee56c23e79a3, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b78106a-b661-444a-a40f-011ae4f0911a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:56.458010691Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:56.458062103Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2b78106a-b661-444a-a40f-011ae4f0911a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f64b0db4-4e9f-4666-b267-c9abe9f8b189, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:56.537168358Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:56.53721666Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f64b0db4-4e9f-4666-b267-c9abe9f8b189, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56d79ac6-bb86-49e7-905e-343e52dbbb90, dataset_name: DatasetName { pool_name: ZpoolName { id: bd06e234-5970-4365-a0e5-6cd29811daea (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:56.621169238Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:56.621221479Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 56d79ac6-bb86-49e7-905e-343e52dbbb90, dataset_name: DatasetName { pool_name: ZpoolName { id: bd06e234-5970-4365-a0e5-6cd29811daea (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 010164b7-56ba-4dda-8f67-3d5e05751e2e, dataset_name: DatasetName { pool_name: ZpoolName { id: e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:56.703888428Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:56.703937949Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 010164b7-56ba-4dda-8f67-3d5e05751e2e, dataset_name: DatasetName { pool_name: ZpoolName { id: e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3488ba32-8748-4793-9389-5ea882fa963c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:56.795626998Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:56.795679949Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3488ba32-8748-4793-9389-5ea882fa963c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89e132b1-d133-478e-a8f7-cf9f30b211c9, dataset_name: DatasetName { pool_name: ZpoolName { id: cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:56.880053008Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:56.880102939Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 89e132b1-d133-478e-a8f7-cf9f30b211c9, dataset_name: DatasetName { pool_name: ZpoolName { id: cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:56.965229365Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/968bbfe8-8122-4a85-96dd-954188a2f89d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:56.965391039Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/e8396e20-0493-4030-b55b-5c9e59a237cf/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:56.965811668Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:56.970097468Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:57.087862006Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:57.087913137Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:57.087919697Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:57.087924337Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:57.087937867Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:57.087942248Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:57.087946578Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:57.087950768Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:57.087955888Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:57.087960198Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_b202e971-de3b-4a73-a534-ee56c23e79a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:57.087964578Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:57.087967778Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:57.087971038Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:57.087974218Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:57.087977508Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:57.087980719Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:57.087983979Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:57.088124532Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_f09c8d14-475d-49d8-b03f-d3a0b7b880cc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:57.088132252Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:57.088395878Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"2cc5212b-858b-4f18-baea-6f5ef3b11c52","remote_addr":"[fd00:1122:3344:101::1]:39162","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02\", TimeNotSynchronized), (\"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3\", TimeNotSynchronized), (\"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752\", TimeNotSynchronized), (\"oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139\", TimeNotSynchronized), (\"oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4\", TimeNotSynchronized), (\"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717\", TimeNotSynchronized), (\"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93\", TimeNotSynchronized), (\"oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a\", TimeNotSynchronized), (\"oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9\", TimeNotSynchronized), (\"oxz_crucible_b202e971-de3b-4a73-a534-ee56c23e79a3\", TimeNotSynchronized), (\"oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e\", TimeNotSynchronized), (\"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075\", TimeNotSynchronized), (\"oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02\", TimeNotSynchronized), (\"oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c\", TimeNotSynchronized), (\"oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189\", TimeNotSynchronized), (\"oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94\", TimeNotSynchronized), (\"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f\", TimeNotSynchronized), (\"oxz_crucible_f09c8d14-475d-49d8-b03f-d3a0b7b880cc\", TimeNotSynchronized), (\"oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90\", TimeNotSynchronized)]","latency_us":1322427,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:57.088749597Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"96.241218ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"2cc5212b-858b-4f18-baea-6f5ef3b11c52\", \"content-length\": \"124\", \"date\": \"Wed, 26 Jun 2024 05:35:57 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"2cc5212b-858b-4f18-baea-6f5ef3b11c52\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 9f65db5e-d949-4342-9d90-6d691adea120, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: a4d358ae-9d04-40a4-b4a5-ac8418ace88d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 707beb4d-acae-4ff1-bb7e-97992526c5fb, kind: Service { id: a4d358ae-9d04-40a4-b4a5-ac8418ace88d }, name: Name(\"ntp-a4d358ae-9d04-40a4-b4a5-ac8418ace88d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 238, 27])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e3552cf6-27c7-4100-942b-c5877c673046, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } } } }, OmicronZoneConfig { id: 9a0125d1-b636-4039-9e54-dea22726a38a, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } } } }, OmicronZoneConfig { id: 7c851202-bc92-4a32-b946-68379ad242d7, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 653e8bac-3f5b-49fe-a9f9-e9f0546059e9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External } } } }, OmicronZoneConfig { id: e4ffe9aa-395e-4536-a4d0-5cf0bc814584, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 48ce280c-ebea-49f3-9829-dc9b4d018a02, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: c2791662-4c87-4a63-a67a-4d6489fb4eaf, kind: Service { id: 48ce280c-ebea-49f3-9829-dc9b4d018a02 }, name: Name(\"external-dns-48ce280c-ebea-49f3-9829-dc9b4d018a02\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 86])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 38cec1c6-d097-4604-b9cb-30a5fab57e35, kind: Service { id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94 }, name: Name(\"external-dns-9f29eb53-b3d5-4fc1-b4d8-a252757e0c94\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 215])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 67179f00-d639-4f5b-b3c6-71d676113139, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: a724cf8c-c466-4981-aa80-a17ab2349f83, kind: Service { id: 67179f00-d639-4f5b-b3c6-71d676113139 }, name: Name(\"nexus-67179f00-d639-4f5b-b3c6-71d676113139\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 55])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7e2a6156-04b4-4797-8bed-d0c5c9cbe752, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: e31530f0-72e9-4544-bfbb-847c405d46c8, kind: Service { id: 7e2a6156-04b4-4797-8bed-d0c5c9cbe752 }, name: Name(\"nexus-7e2a6156-04b4-4797-8bed-d0c5c9cbe752\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 20a0479c-19b7-40a0-a935-5ff5399f7d02, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 79c95d4d-cff5-4419-8f3e-a6e216736b73, kind: Service { id: 20a0479c-19b7-40a0-a935-5ff5399f7d02 }, name: Name(\"nexus-20a0479c-19b7-40a0-a935-5ff5399f7d02\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 249])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: e42e8525-00d2-488f-a732-5c9514a5c7b4, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 96bb9301-f50f-4259-8a1b-169ec98cbff3, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } } } }, OmicronZoneConfig { id: 966bee63-3642-4e7a-a159-17eb59b72717, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: da8be60e-c115-4289-8526-a1ad7b366b93, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: d72762b4-6a43-4e1c-92b1-b432c1731075, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: c3d43ade-9569-403a-a6cc-494ec947793f, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } } } }, OmicronZoneConfig { id: f09c8d14-475d-49d8-b03f-d3a0b7b880cc, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } } } }, OmicronZoneConfig { id: b202e971-de3b-4a73-a534-ee56c23e79a3, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2b78106a-b661-444a-a40f-011ae4f0911a, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External } } } }, OmicronZoneConfig { id: f64b0db4-4e9f-4666-b267-c9abe9f8b189, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 56d79ac6-bb86-49e7-905e-343e52dbbb90, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd06e234-5970-4365-a0e5-6cd29811daea (zpool), kind: External } } } }, OmicronZoneConfig { id: 010164b7-56ba-4dda-8f67-3d5e05751e2e, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3488ba32-8748-4793-9389-5ea882fa963c, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool), kind: External } } } }, OmicronZoneConfig { id: 89e132b1-d133-478e-a8f7-cf9f30b211c9, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:57.186412705Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:57.187450899Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:57.187460689Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9f65db5e-d949-4342-9d90-6d691adea120, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:57.209466693Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:57.209499603Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9f65db5e-d949-4342-9d90-6d691adea120, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:57.231083697Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:57.231115548Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e3552cf6-27c7-4100-942b-c5877c673046, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:57.251838271Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:57.251867342Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e3552cf6-27c7-4100-942b-c5877c673046, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a0125d1-b636-4039-9e54-dea22726a38a, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:57.27233784Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:57.27236513Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a0125d1-b636-4039-9e54-dea22726a38a, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c851202-bc92-4a32-b946-68379ad242d7, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:57.292557851Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:57.292587032Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7c851202-bc92-4a32-b946-68379ad242d7, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 653e8bac-3f5b-49fe-a9f9-e9f0546059e9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:57.313212113Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:57.313237004Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 653e8bac-3f5b-49fe-a9f9-e9f0546059e9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e4ffe9aa-395e-4536-a4d0-5cf0bc814584, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:57.332494353Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:57.332528944Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e4ffe9aa-395e-4536-a4d0-5cf0bc814584, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48ce280c-ebea-49f3-9829-dc9b4d018a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:57.351369343Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:57.351405494Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 48ce280c-ebea-49f3-9829-dc9b4d018a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:57.370758239Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:57.370783777Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 96bb9301-f50f-4259-8a1b-169ec98cbff3, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:57.390898213Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:57.390921913Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 96bb9301-f50f-4259-8a1b-169ec98cbff3, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c3d43ade-9569-403a-a6cc-494ec947793f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:57.410501783Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:57.410529413Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c3d43ade-9569-403a-a6cc-494ec947793f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f09c8d14-475d-49d8-b03f-d3a0b7b880cc, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:57.429063816Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:57.429089366Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f09c8d14-475d-49d8-b03f-d3a0b7b880cc, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b202e971-de3b-4a73-a534-ee56c23e79a3, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:57.449976274Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:57.450007794Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b202e971-de3b-4a73-a534-ee56c23e79a3, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b78106a-b661-444a-a40f-011ae4f0911a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:57.471302241Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:57.471328052Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2b78106a-b661-444a-a40f-011ae4f0911a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f64b0db4-4e9f-4666-b267-c9abe9f8b189, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:57.490823347Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:57.490853067Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f64b0db4-4e9f-4666-b267-c9abe9f8b189, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56d79ac6-bb86-49e7-905e-343e52dbbb90, dataset_name: DatasetName { pool_name: ZpoolName { id: bd06e234-5970-4365-a0e5-6cd29811daea (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:57.509719847Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:57.509746158Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 56d79ac6-bb86-49e7-905e-343e52dbbb90, dataset_name: DatasetName { pool_name: ZpoolName { id: bd06e234-5970-4365-a0e5-6cd29811daea (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 010164b7-56ba-4dda-8f67-3d5e05751e2e, dataset_name: DatasetName { pool_name: ZpoolName { id: e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:57.527949233Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:57.527982713Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 010164b7-56ba-4dda-8f67-3d5e05751e2e, dataset_name: DatasetName { pool_name: ZpoolName { id: e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3488ba32-8748-4793-9389-5ea882fa963c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:57.548016771Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:57.548050422Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3488ba32-8748-4793-9389-5ea882fa963c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89e132b1-d133-478e-a8f7-cf9f30b211c9, dataset_name: DatasetName { pool_name: ZpoolName { id: cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:57.567017264Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:57.567044655Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 89e132b1-d133-478e-a8f7-cf9f30b211c9, dataset_name: DatasetName { pool_name: ZpoolName { id: cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:57.586418197Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/968bbfe8-8122-4a85-96dd-954188a2f89d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:57.586457538Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/e8396e20-0493-4030-b55b-5c9e59a237cf/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:57.586708064Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:57.58698837Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:57.689682146Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:57.689733227Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_b202e971-de3b-4a73-a534-ee56c23e79a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:57.689742007Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:57.689748158Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_f09c8d14-475d-49d8-b03f-d3a0b7b880cc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:57.689770418Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:57.689776408Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:57.689781738Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:57.689787528Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:57.689794699Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:57.689799759Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:57.689805029Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:57.689811949Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:57.689817489Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:57.689822809Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:57.689827909Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:57.689834869Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:57.68983989Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:57.68984539Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:57.68985073Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:57.689955122Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"49a14d15-a264-468e-912b-f7b2a5abcc76","remote_addr":"[fd00:1122:3344:101::1]:39162","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752\", TimeNotSynchronized), (\"oxz_crucible_b202e971-de3b-4a73-a534-ee56c23e79a3\", TimeNotSynchronized), (\"oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a\", TimeNotSynchronized), (\"oxz_crucible_f09c8d14-475d-49d8-b03f-d3a0b7b880cc\", TimeNotSynchronized), (\"oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e\", TimeNotSynchronized), (\"oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9\", TimeNotSynchronized), (\"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075\", TimeNotSynchronized), (\"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3\", TimeNotSynchronized), (\"oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139\", TimeNotSynchronized), (\"oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189\", TimeNotSynchronized), (\"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93\", TimeNotSynchronized), (\"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717\", TimeNotSynchronized), (\"oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94\", TimeNotSynchronized), (\"oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02\", TimeNotSynchronized), (\"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f\", TimeNotSynchronized), (\"oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90\", TimeNotSynchronized), (\"oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4\", TimeNotSynchronized), (\"oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c\", TimeNotSynchronized), (\"oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02\", TimeNotSynchronized)]","latency_us":502785,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:57.69027998Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"146.184846ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"49a14d15-a264-468e-912b-f7b2a5abcc76\", \"content-length\": \"124\", \"date\": \"Wed, 26 Jun 2024 05:35:57 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"49a14d15-a264-468e-912b-f7b2a5abcc76\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 9f65db5e-d949-4342-9d90-6d691adea120, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: a4d358ae-9d04-40a4-b4a5-ac8418ace88d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 707beb4d-acae-4ff1-bb7e-97992526c5fb, kind: Service { id: a4d358ae-9d04-40a4-b4a5-ac8418ace88d }, name: Name(\"ntp-a4d358ae-9d04-40a4-b4a5-ac8418ace88d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 238, 27])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e3552cf6-27c7-4100-942b-c5877c673046, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } } } }, OmicronZoneConfig { id: 9a0125d1-b636-4039-9e54-dea22726a38a, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } } } }, OmicronZoneConfig { id: 7c851202-bc92-4a32-b946-68379ad242d7, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 653e8bac-3f5b-49fe-a9f9-e9f0546059e9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External } } } }, OmicronZoneConfig { id: e4ffe9aa-395e-4536-a4d0-5cf0bc814584, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 48ce280c-ebea-49f3-9829-dc9b4d018a02, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: c2791662-4c87-4a63-a67a-4d6489fb4eaf, kind: Service { id: 48ce280c-ebea-49f3-9829-dc9b4d018a02 }, name: Name(\"external-dns-48ce280c-ebea-49f3-9829-dc9b4d018a02\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 86])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 38cec1c6-d097-4604-b9cb-30a5fab57e35, kind: Service { id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94 }, name: Name(\"external-dns-9f29eb53-b3d5-4fc1-b4d8-a252757e0c94\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 215])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 67179f00-d639-4f5b-b3c6-71d676113139, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: a724cf8c-c466-4981-aa80-a17ab2349f83, kind: Service { id: 67179f00-d639-4f5b-b3c6-71d676113139 }, name: Name(\"nexus-67179f00-d639-4f5b-b3c6-71d676113139\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 55])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7e2a6156-04b4-4797-8bed-d0c5c9cbe752, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: e31530f0-72e9-4544-bfbb-847c405d46c8, kind: Service { id: 7e2a6156-04b4-4797-8bed-d0c5c9cbe752 }, name: Name(\"nexus-7e2a6156-04b4-4797-8bed-d0c5c9cbe752\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 20a0479c-19b7-40a0-a935-5ff5399f7d02, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 79c95d4d-cff5-4419-8f3e-a6e216736b73, kind: Service { id: 20a0479c-19b7-40a0-a935-5ff5399f7d02 }, name: Name(\"nexus-20a0479c-19b7-40a0-a935-5ff5399f7d02\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 249])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: e42e8525-00d2-488f-a732-5c9514a5c7b4, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 96bb9301-f50f-4259-8a1b-169ec98cbff3, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } } } }, OmicronZoneConfig { id: 966bee63-3642-4e7a-a159-17eb59b72717, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: da8be60e-c115-4289-8526-a1ad7b366b93, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: d72762b4-6a43-4e1c-92b1-b432c1731075, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: c3d43ade-9569-403a-a6cc-494ec947793f, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } } } }, OmicronZoneConfig { id: f09c8d14-475d-49d8-b03f-d3a0b7b880cc, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } } } }, OmicronZoneConfig { id: b202e971-de3b-4a73-a534-ee56c23e79a3, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2b78106a-b661-444a-a40f-011ae4f0911a, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External } } } }, OmicronZoneConfig { id: f64b0db4-4e9f-4666-b267-c9abe9f8b189, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 56d79ac6-bb86-49e7-905e-343e52dbbb90, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd06e234-5970-4365-a0e5-6cd29811daea (zpool), kind: External } } } }, OmicronZoneConfig { id: 010164b7-56ba-4dda-8f67-3d5e05751e2e, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3488ba32-8748-4793-9389-5ea882fa963c, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool), kind: External } } } }, OmicronZoneConfig { id: 89e132b1-d133-478e-a8f7-cf9f30b211c9, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:57.837513095Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:57.837818543Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:57.837830029Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9f65db5e-d949-4342-9d90-6d691adea120, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:57.859654341Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:57.859699543Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9f65db5e-d949-4342-9d90-6d691adea120, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:57.880519838Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:57.880561199Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e3552cf6-27c7-4100-942b-c5877c673046, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:57.900904124Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:57.900957665Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e3552cf6-27c7-4100-942b-c5877c673046, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a0125d1-b636-4039-9e54-dea22726a38a, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:57.921247788Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:57.921287199Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a0125d1-b636-4039-9e54-dea22726a38a, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c851202-bc92-4a32-b946-68379ad242d7, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:57.94144836Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:57.941487571Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7c851202-bc92-4a32-b946-68379ad242d7, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 653e8bac-3f5b-49fe-a9f9-e9f0546059e9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:57.962116232Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:57.962155163Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 653e8bac-3f5b-49fe-a9f9-e9f0546059e9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e4ffe9aa-395e-4536-a4d0-5cf0bc814584, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:57.982382115Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:57.982421835Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e4ffe9aa-395e-4536-a4d0-5cf0bc814584, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48ce280c-ebea-49f3-9829-dc9b4d018a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:58.002484504Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:58.002554535Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 48ce280c-ebea-49f3-9829-dc9b4d018a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:58.024226791Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:58.024268662Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 96bb9301-f50f-4259-8a1b-169ec98cbff3, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:58.046405358Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:58.046452799Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 96bb9301-f50f-4259-8a1b-169ec98cbff3, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c3d43ade-9569-403a-a6cc-494ec947793f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:58.066573159Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:58.06662068Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c3d43ade-9569-403a-a6cc-494ec947793f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f09c8d14-475d-49d8-b03f-d3a0b7b880cc, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:58.088611503Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:58.088652874Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f09c8d14-475d-49d8-b03f-d3a0b7b880cc, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b202e971-de3b-4a73-a534-ee56c23e79a3, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:58.108582809Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:58.1086251Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b202e971-de3b-4a73-a534-ee56c23e79a3, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b78106a-b661-444a-a40f-011ae4f0911a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:58.127681364Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:58.127718665Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2b78106a-b661-444a-a40f-011ae4f0911a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f64b0db4-4e9f-4666-b267-c9abe9f8b189, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:58.148592382Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:58.148630753Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f64b0db4-4e9f-4666-b267-c9abe9f8b189, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56d79ac6-bb86-49e7-905e-343e52dbbb90, dataset_name: DatasetName { pool_name: ZpoolName { id: bd06e234-5970-4365-a0e5-6cd29811daea (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:58.168679001Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:58.168718732Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 56d79ac6-bb86-49e7-905e-343e52dbbb90, dataset_name: DatasetName { pool_name: ZpoolName { id: bd06e234-5970-4365-a0e5-6cd29811daea (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 010164b7-56ba-4dda-8f67-3d5e05751e2e, dataset_name: DatasetName { pool_name: ZpoolName { id: e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:58.189381964Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:58.189452316Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 010164b7-56ba-4dda-8f67-3d5e05751e2e, dataset_name: DatasetName { pool_name: ZpoolName { id: e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3488ba32-8748-4793-9389-5ea882fa963c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:58.209459432Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:58.209497043Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3488ba32-8748-4793-9389-5ea882fa963c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89e132b1-d133-478e-a8f7-cf9f30b211c9, dataset_name: DatasetName { pool_name: ZpoolName { id: cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:58.228928797Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:58.228971258Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 89e132b1-d133-478e-a8f7-cf9f30b211c9, dataset_name: DatasetName { pool_name: ZpoolName { id: cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:58.251892392Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/968bbfe8-8122-4a85-96dd-954188a2f89d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:58.251944943Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/e8396e20-0493-4030-b55b-5c9e59a237cf/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:58.252167289Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:58.252337923Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:58.350562354Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:58.350607865Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:58.350614495Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:58.350618165Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:58.350630325Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:58.350633856Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:58.350637226Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:58.350640556Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:58.350646666Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_f09c8d14-475d-49d8-b03f-d3a0b7b880cc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:58.350649896Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:58.350653066Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:58.350756019Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:58.350763729Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:58.350771899Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:58.350782719Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:58.350789519Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:58.35080016Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_b202e971-de3b-4a73-a534-ee56c23e79a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:58.35081815Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:58.3508226Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:58.351074736Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"51c32445-1cf0-4d9f-9a7f-2441e1f63dcb","remote_addr":"[fd00:1122:3344:101::1]:39162","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90\", TimeNotSynchronized), (\"oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94\", TimeNotSynchronized), (\"oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c\", TimeNotSynchronized), (\"oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9\", TimeNotSynchronized), (\"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f\", TimeNotSynchronized), (\"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075\", TimeNotSynchronized), (\"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717\", TimeNotSynchronized), (\"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93\", TimeNotSynchronized), (\"oxz_crucible_f09c8d14-475d-49d8-b03f-d3a0b7b880cc\", TimeNotSynchronized), (\"oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4\", TimeNotSynchronized), (\"oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a\", TimeNotSynchronized), (\"oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02\", TimeNotSynchronized), (\"oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139\", TimeNotSynchronized), (\"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752\", TimeNotSynchronized), (\"oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02\", TimeNotSynchronized), (\"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3\", TimeNotSynchronized), (\"oxz_crucible_b202e971-de3b-4a73-a534-ee56c23e79a3\", TimeNotSynchronized), (\"oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189\", TimeNotSynchronized), (\"oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e\", TimeNotSynchronized)]","latency_us":513257,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:58.35125538Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"201.45269ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"51c32445-1cf0-4d9f-9a7f-2441e1f63dcb\", \"content-length\": \"124\", \"date\": \"Wed, 26 Jun 2024 05:35:58 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"51c32445-1cf0-4d9f-9a7f-2441e1f63dcb\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 9f65db5e-d949-4342-9d90-6d691adea120, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: a4d358ae-9d04-40a4-b4a5-ac8418ace88d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 707beb4d-acae-4ff1-bb7e-97992526c5fb, kind: Service { id: a4d358ae-9d04-40a4-b4a5-ac8418ace88d }, name: Name(\"ntp-a4d358ae-9d04-40a4-b4a5-ac8418ace88d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 238, 27])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e3552cf6-27c7-4100-942b-c5877c673046, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } } } }, OmicronZoneConfig { id: 9a0125d1-b636-4039-9e54-dea22726a38a, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } } } }, OmicronZoneConfig { id: 7c851202-bc92-4a32-b946-68379ad242d7, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 653e8bac-3f5b-49fe-a9f9-e9f0546059e9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External } } } }, OmicronZoneConfig { id: e4ffe9aa-395e-4536-a4d0-5cf0bc814584, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 48ce280c-ebea-49f3-9829-dc9b4d018a02, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: c2791662-4c87-4a63-a67a-4d6489fb4eaf, kind: Service { id: 48ce280c-ebea-49f3-9829-dc9b4d018a02 }, name: Name(\"external-dns-48ce280c-ebea-49f3-9829-dc9b4d018a02\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 86])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 38cec1c6-d097-4604-b9cb-30a5fab57e35, kind: Service { id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94 }, name: Name(\"external-dns-9f29eb53-b3d5-4fc1-b4d8-a252757e0c94\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 215])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 67179f00-d639-4f5b-b3c6-71d676113139, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: a724cf8c-c466-4981-aa80-a17ab2349f83, kind: Service { id: 67179f00-d639-4f5b-b3c6-71d676113139 }, name: Name(\"nexus-67179f00-d639-4f5b-b3c6-71d676113139\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 55])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7e2a6156-04b4-4797-8bed-d0c5c9cbe752, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: e31530f0-72e9-4544-bfbb-847c405d46c8, kind: Service { id: 7e2a6156-04b4-4797-8bed-d0c5c9cbe752 }, name: Name(\"nexus-7e2a6156-04b4-4797-8bed-d0c5c9cbe752\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 20a0479c-19b7-40a0-a935-5ff5399f7d02, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 79c95d4d-cff5-4419-8f3e-a6e216736b73, kind: Service { id: 20a0479c-19b7-40a0-a935-5ff5399f7d02 }, name: Name(\"nexus-20a0479c-19b7-40a0-a935-5ff5399f7d02\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 249])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: e42e8525-00d2-488f-a732-5c9514a5c7b4, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 96bb9301-f50f-4259-8a1b-169ec98cbff3, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } } } }, OmicronZoneConfig { id: 966bee63-3642-4e7a-a159-17eb59b72717, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: da8be60e-c115-4289-8526-a1ad7b366b93, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: d72762b4-6a43-4e1c-92b1-b432c1731075, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: c3d43ade-9569-403a-a6cc-494ec947793f, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } } } }, OmicronZoneConfig { id: f09c8d14-475d-49d8-b03f-d3a0b7b880cc, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } } } }, OmicronZoneConfig { id: b202e971-de3b-4a73-a534-ee56c23e79a3, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2b78106a-b661-444a-a40f-011ae4f0911a, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External } } } }, OmicronZoneConfig { id: f64b0db4-4e9f-4666-b267-c9abe9f8b189, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 56d79ac6-bb86-49e7-905e-343e52dbbb90, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd06e234-5970-4365-a0e5-6cd29811daea (zpool), kind: External } } } }, OmicronZoneConfig { id: 010164b7-56ba-4dda-8f67-3d5e05751e2e, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3488ba32-8748-4793-9389-5ea882fa963c, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool), kind: External } } } }, OmicronZoneConfig { id: 89e132b1-d133-478e-a8f7-cf9f30b211c9, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:58.553695863Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:58.554574474Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:58.554583284Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9f65db5e-d949-4342-9d90-6d691adea120, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:58.577284653Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:58.577323164Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9f65db5e-d949-4342-9d90-6d691adea120, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:58.59772234Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:58.597761701Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e3552cf6-27c7-4100-942b-c5877c673046, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:58.617132663Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:58.617181734Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e3552cf6-27c7-4100-942b-c5877c673046, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a0125d1-b636-4039-9e54-dea22726a38a, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:58.636782221Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:58.636817582Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a0125d1-b636-4039-9e54-dea22726a38a, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c851202-bc92-4a32-b946-68379ad242d7, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:58.656370448Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:58.656410519Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7c851202-bc92-4a32-b946-68379ad242d7, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 653e8bac-3f5b-49fe-a9f9-e9f0546059e9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:58.682261278Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:58.682296726Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 653e8bac-3f5b-49fe-a9f9-e9f0546059e9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e4ffe9aa-395e-4536-a4d0-5cf0bc814584, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:58.706381295Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:58.706432836Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e4ffe9aa-395e-4536-a4d0-5cf0bc814584, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48ce280c-ebea-49f3-9829-dc9b4d018a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:58.7310066Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:58.731051331Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 48ce280c-ebea-49f3-9829-dc9b4d018a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:58.758250715Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:58.758300826Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 96bb9301-f50f-4259-8a1b-169ec98cbff3, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:58.786693099Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:58.78673437Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 96bb9301-f50f-4259-8a1b-169ec98cbff3, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c3d43ade-9569-403a-a6cc-494ec947793f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:58.813935514Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:58.813974645Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c3d43ade-9569-403a-a6cc-494ec947793f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f09c8d14-475d-49d8-b03f-d3a0b7b880cc, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:58.844121029Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:58.844161019Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f09c8d14-475d-49d8-b03f-d3a0b7b880cc, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b202e971-de3b-4a73-a534-ee56c23e79a3, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:58.874383044Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:58.874420316Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b202e971-de3b-4a73-a534-ee56c23e79a3, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b78106a-b661-444a-a40f-011ae4f0911a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:58.900586866Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:58.900623167Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2b78106a-b661-444a-a40f-011ae4f0911a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f64b0db4-4e9f-4666-b267-c9abe9f8b189, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:58.926882795Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:58.92691917Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f64b0db4-4e9f-4666-b267-c9abe9f8b189, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56d79ac6-bb86-49e7-905e-343e52dbbb90, dataset_name: DatasetName { pool_name: ZpoolName { id: bd06e234-5970-4365-a0e5-6cd29811daea (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:58.957852182Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:58.957896013Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 56d79ac6-bb86-49e7-905e-343e52dbbb90, dataset_name: DatasetName { pool_name: ZpoolName { id: bd06e234-5970-4365-a0e5-6cd29811daea (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 010164b7-56ba-4dda-8f67-3d5e05751e2e, dataset_name: DatasetName { pool_name: ZpoolName { id: e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:58.988288709Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:58.988337366Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 010164b7-56ba-4dda-8f67-3d5e05751e2e, dataset_name: DatasetName { pool_name: ZpoolName { id: e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3488ba32-8748-4793-9389-5ea882fa963c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:59.016190453Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:59.016232374Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3488ba32-8748-4793-9389-5ea882fa963c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89e132b1-d133-478e-a8f7-cf9f30b211c9, dataset_name: DatasetName { pool_name: ZpoolName { id: cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:59.046698365Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:59.046735845Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 89e132b1-d133-478e-a8f7-cf9f30b211c9, dataset_name: DatasetName { pool_name: ZpoolName { id: cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:59.070691204Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/968bbfe8-8122-4a85-96dd-954188a2f89d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:59.070737265Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/e8396e20-0493-4030-b55b-5c9e59a237cf/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:59.070985871Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:59.071140135Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:59.184008448Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_f09c8d14-475d-49d8-b03f-d3a0b7b880cc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:59.184064549Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:59.184072149Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:59.18410239Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:59.18411039Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:59.184119381Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:59.184126251Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:59.184131561Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_b202e971-de3b-4a73-a534-ee56c23e79a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:59.184137121Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:59.184144351Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:59.184149821Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:59.184156511Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:59.184164522Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:59.184170032Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:59.184176692Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:59.184182112Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:59.184188892Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:59.184195542Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:59.184200623Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:59.184302065Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"3f241c3d-6606-4a99-8428-d31a1d04f575","remote_addr":"[fd00:1122:3344:101::1]:39162","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_f09c8d14-475d-49d8-b03f-d3a0b7b880cc\", TimeNotSynchronized), (\"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93\", TimeNotSynchronized), (\"oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4\", TimeNotSynchronized), (\"oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a\", TimeNotSynchronized), (\"oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189\", TimeNotSynchronized), (\"oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94\", TimeNotSynchronized), (\"oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139\", TimeNotSynchronized), (\"oxz_crucible_b202e971-de3b-4a73-a534-ee56c23e79a3\", TimeNotSynchronized), (\"oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90\", TimeNotSynchronized), (\"oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e\", TimeNotSynchronized), (\"oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9\", TimeNotSynchronized), (\"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717\", TimeNotSynchronized), (\"oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c\", TimeNotSynchronized), (\"oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02\", TimeNotSynchronized), (\"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752\", TimeNotSynchronized), (\"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3\", TimeNotSynchronized), (\"oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02\", TimeNotSynchronized), (\"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075\", TimeNotSynchronized), (\"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f\", TimeNotSynchronized)]","latency_us":630387,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:59.18450703Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"181.865646ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3f241c3d-6606-4a99-8428-d31a1d04f575\", \"content-length\": \"124\", \"date\": \"Wed, 26 Jun 2024 05:35:58 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"3f241c3d-6606-4a99-8428-d31a1d04f575\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 9f65db5e-d949-4342-9d90-6d691adea120, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: a4d358ae-9d04-40a4-b4a5-ac8418ace88d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 707beb4d-acae-4ff1-bb7e-97992526c5fb, kind: Service { id: a4d358ae-9d04-40a4-b4a5-ac8418ace88d }, name: Name(\"ntp-a4d358ae-9d04-40a4-b4a5-ac8418ace88d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 238, 27])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e3552cf6-27c7-4100-942b-c5877c673046, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } } } }, OmicronZoneConfig { id: 9a0125d1-b636-4039-9e54-dea22726a38a, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } } } }, OmicronZoneConfig { id: 7c851202-bc92-4a32-b946-68379ad242d7, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 653e8bac-3f5b-49fe-a9f9-e9f0546059e9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External } } } }, OmicronZoneConfig { id: e4ffe9aa-395e-4536-a4d0-5cf0bc814584, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 48ce280c-ebea-49f3-9829-dc9b4d018a02, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: c2791662-4c87-4a63-a67a-4d6489fb4eaf, kind: Service { id: 48ce280c-ebea-49f3-9829-dc9b4d018a02 }, name: Name(\"external-dns-48ce280c-ebea-49f3-9829-dc9b4d018a02\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 86])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 38cec1c6-d097-4604-b9cb-30a5fab57e35, kind: Service { id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94 }, name: Name(\"external-dns-9f29eb53-b3d5-4fc1-b4d8-a252757e0c94\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 215])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 67179f00-d639-4f5b-b3c6-71d676113139, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: a724cf8c-c466-4981-aa80-a17ab2349f83, kind: Service { id: 67179f00-d639-4f5b-b3c6-71d676113139 }, name: Name(\"nexus-67179f00-d639-4f5b-b3c6-71d676113139\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 55])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7e2a6156-04b4-4797-8bed-d0c5c9cbe752, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: e31530f0-72e9-4544-bfbb-847c405d46c8, kind: Service { id: 7e2a6156-04b4-4797-8bed-d0c5c9cbe752 }, name: Name(\"nexus-7e2a6156-04b4-4797-8bed-d0c5c9cbe752\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 20a0479c-19b7-40a0-a935-5ff5399f7d02, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 79c95d4d-cff5-4419-8f3e-a6e216736b73, kind: Service { id: 20a0479c-19b7-40a0-a935-5ff5399f7d02 }, name: Name(\"nexus-20a0479c-19b7-40a0-a935-5ff5399f7d02\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 249])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: e42e8525-00d2-488f-a732-5c9514a5c7b4, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 96bb9301-f50f-4259-8a1b-169ec98cbff3, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } } } }, OmicronZoneConfig { id: 966bee63-3642-4e7a-a159-17eb59b72717, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: da8be60e-c115-4289-8526-a1ad7b366b93, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: d72762b4-6a43-4e1c-92b1-b432c1731075, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: c3d43ade-9569-403a-a6cc-494ec947793f, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } } } }, OmicronZoneConfig { id: f09c8d14-475d-49d8-b03f-d3a0b7b880cc, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } } } }, OmicronZoneConfig { id: b202e971-de3b-4a73-a534-ee56c23e79a3, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2b78106a-b661-444a-a40f-011ae4f0911a, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External } } } }, OmicronZoneConfig { id: f64b0db4-4e9f-4666-b267-c9abe9f8b189, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 56d79ac6-bb86-49e7-905e-343e52dbbb90, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd06e234-5970-4365-a0e5-6cd29811daea (zpool), kind: External } } } }, OmicronZoneConfig { id: 010164b7-56ba-4dda-8f67-3d5e05751e2e, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3488ba32-8748-4793-9389-5ea882fa963c, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool), kind: External } } } }, OmicronZoneConfig { id: 89e132b1-d133-478e-a8f7-cf9f30b211c9, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:59.367989469Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:59.368391719Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:59.368400609Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9f65db5e-d949-4342-9d90-6d691adea120, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:59.396126036Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:59.396184027Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9f65db5e-d949-4342-9d90-6d691adea120, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:59.422425019Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:59.42246574Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e3552cf6-27c7-4100-942b-c5877c673046, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:59.450051554Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:59.450105175Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e3552cf6-27c7-4100-942b-c5877c673046, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a0125d1-b636-4039-9e54-dea22726a38a, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:59.473277916Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:59.473323097Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a0125d1-b636-4039-9e54-dea22726a38a, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c851202-bc92-4a32-b946-68379ad242d7, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:59.498449473Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:59.498491824Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7c851202-bc92-4a32-b946-68379ad242d7, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 653e8bac-3f5b-49fe-a9f9-e9f0546059e9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:59.52103151Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:59.521085741Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 653e8bac-3f5b-49fe-a9f9-e9f0546059e9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e4ffe9aa-395e-4536-a4d0-5cf0bc814584, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:59.544807604Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:59.544854736Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e4ffe9aa-395e-4536-a4d0-5cf0bc814584, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48ce280c-ebea-49f3-9829-dc9b4d018a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:59.569581113Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:59.569626234Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 48ce280c-ebea-49f3-9829-dc9b4d018a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:59.595966358Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:59.596015539Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 96bb9301-f50f-4259-8a1b-169ec98cbff3, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:59.620761607Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:59.620802408Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 96bb9301-f50f-4259-8a1b-169ec98cbff3, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c3d43ade-9569-403a-a6cc-494ec947793f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:59.646590939Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:59.6466381Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c3d43ade-9569-403a-a6cc-494ec947793f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f09c8d14-475d-49d8-b03f-d3a0b7b880cc, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:59.673777133Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:59.673817724Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f09c8d14-475d-49d8-b03f-d3a0b7b880cc, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b202e971-de3b-4a73-a534-ee56c23e79a3, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:59.699575397Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:59.699616487Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b202e971-de3b-4a73-a534-ee56c23e79a3, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b78106a-b661-444a-a40f-011ae4f0911a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:59.725617953Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:59.725660494Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2b78106a-b661-444a-a40f-011ae4f0911a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f64b0db4-4e9f-4666-b267-c9abe9f8b189, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:59.751066667Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:59.751111108Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f64b0db4-4e9f-4666-b267-c9abe9f8b189, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56d79ac6-bb86-49e7-905e-343e52dbbb90, dataset_name: DatasetName { pool_name: ZpoolName { id: bd06e234-5970-4365-a0e5-6cd29811daea (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:59.776867858Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:59.77691997Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 56d79ac6-bb86-49e7-905e-343e52dbbb90, dataset_name: DatasetName { pool_name: ZpoolName { id: bd06e234-5970-4365-a0e5-6cd29811daea (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 010164b7-56ba-4dda-8f67-3d5e05751e2e, dataset_name: DatasetName { pool_name: ZpoolName { id: e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:59.802112577Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:59.802266341Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 010164b7-56ba-4dda-8f67-3d5e05751e2e, dataset_name: DatasetName { pool_name: ZpoolName { id: e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3488ba32-8748-4793-9389-5ea882fa963c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:59.825949503Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:59.825994704Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3488ba32-8748-4793-9389-5ea882fa963c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89e132b1-d133-478e-a8f7-cf9f30b211c9, dataset_name: DatasetName { pool_name: ZpoolName { id: cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:59.848609012Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:59.848655713Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 89e132b1-d133-478e-a8f7-cf9f30b211c9, dataset_name: DatasetName { pool_name: ZpoolName { id: cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:59.872478808Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/968bbfe8-8122-4a85-96dd-954188a2f89d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:59.872528412Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/e8396e20-0493-4030-b55b-5c9e59a237cf/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:59.872810027Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:59.87295349Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:59.984190961Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:59.984235812Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:59.984241484Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:59.984255847Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:59.984260122Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:59.98426525Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:59.984270997Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:59.984275056Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_f09c8d14-475d-49d8-b03f-d3a0b7b880cc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:59.984279978Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:59.984283831Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:59.984290085Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:59.984294136Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:59.984298767Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:59.98430187Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:59.984305854Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:59.984313064Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_b202e971-de3b-4a73-a534-ee56c23e79a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:59.984317207Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:59.984321932Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:59.984338649Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:35:59.984608289Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"1fbf7f23-4d94-4737-8fa5-07e729580c4d","remote_addr":"[fd00:1122:3344:101::1]:39162","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189\", TimeNotSynchronized), (\"oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139\", TimeNotSynchronized), (\"oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02\", TimeNotSynchronized), (\"oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4\", TimeNotSynchronized), (\"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3\", TimeNotSynchronized), (\"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f\", TimeNotSynchronized), (\"oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9\", TimeNotSynchronized), (\"oxz_crucible_f09c8d14-475d-49d8-b03f-d3a0b7b880cc\", TimeNotSynchronized), (\"oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a\", TimeNotSynchronized), (\"oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c\", TimeNotSynchronized), (\"oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02\", TimeNotSynchronized), (\"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752\", TimeNotSynchronized), (\"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717\", TimeNotSynchronized), (\"oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94\", TimeNotSynchronized), (\"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075\", TimeNotSynchronized), (\"oxz_crucible_b202e971-de3b-4a73-a534-ee56c23e79a3\", TimeNotSynchronized), (\"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93\", TimeNotSynchronized), (\"oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90\", TimeNotSynchronized), (\"oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e\", TimeNotSynchronized)]","latency_us":616291,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:35:59.984875931Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"191.285414ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"1fbf7f23-4d94-4737-8fa5-07e729580c4d\", \"content-length\": \"124\", \"date\": \"Wed, 26 Jun 2024 05:35:59 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"1fbf7f23-4d94-4737-8fa5-07e729580c4d\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 9f65db5e-d949-4342-9d90-6d691adea120, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: a4d358ae-9d04-40a4-b4a5-ac8418ace88d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 707beb4d-acae-4ff1-bb7e-97992526c5fb, kind: Service { id: a4d358ae-9d04-40a4-b4a5-ac8418ace88d }, name: Name(\"ntp-a4d358ae-9d04-40a4-b4a5-ac8418ace88d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 238, 27])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e3552cf6-27c7-4100-942b-c5877c673046, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } } } }, OmicronZoneConfig { id: 9a0125d1-b636-4039-9e54-dea22726a38a, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } } } }, OmicronZoneConfig { id: 7c851202-bc92-4a32-b946-68379ad242d7, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 653e8bac-3f5b-49fe-a9f9-e9f0546059e9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External } } } }, OmicronZoneConfig { id: e4ffe9aa-395e-4536-a4d0-5cf0bc814584, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 48ce280c-ebea-49f3-9829-dc9b4d018a02, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: c2791662-4c87-4a63-a67a-4d6489fb4eaf, kind: Service { id: 48ce280c-ebea-49f3-9829-dc9b4d018a02 }, name: Name(\"external-dns-48ce280c-ebea-49f3-9829-dc9b4d018a02\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 86])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 38cec1c6-d097-4604-b9cb-30a5fab57e35, kind: Service { id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94 }, name: Name(\"external-dns-9f29eb53-b3d5-4fc1-b4d8-a252757e0c94\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 215])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 67179f00-d639-4f5b-b3c6-71d676113139, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: a724cf8c-c466-4981-aa80-a17ab2349f83, kind: Service { id: 67179f00-d639-4f5b-b3c6-71d676113139 }, name: Name(\"nexus-67179f00-d639-4f5b-b3c6-71d676113139\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 55])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7e2a6156-04b4-4797-8bed-d0c5c9cbe752, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: e31530f0-72e9-4544-bfbb-847c405d46c8, kind: Service { id: 7e2a6156-04b4-4797-8bed-d0c5c9cbe752 }, name: Name(\"nexus-7e2a6156-04b4-4797-8bed-d0c5c9cbe752\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 20a0479c-19b7-40a0-a935-5ff5399f7d02, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 79c95d4d-cff5-4419-8f3e-a6e216736b73, kind: Service { id: 20a0479c-19b7-40a0-a935-5ff5399f7d02 }, name: Name(\"nexus-20a0479c-19b7-40a0-a935-5ff5399f7d02\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 249])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: e42e8525-00d2-488f-a732-5c9514a5c7b4, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 96bb9301-f50f-4259-8a1b-169ec98cbff3, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } } } }, OmicronZoneConfig { id: 966bee63-3642-4e7a-a159-17eb59b72717, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: da8be60e-c115-4289-8526-a1ad7b366b93, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: d72762b4-6a43-4e1c-92b1-b432c1731075, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: c3d43ade-9569-403a-a6cc-494ec947793f, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } } } }, OmicronZoneConfig { id: f09c8d14-475d-49d8-b03f-d3a0b7b880cc, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } } } }, OmicronZoneConfig { id: b202e971-de3b-4a73-a534-ee56c23e79a3, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2b78106a-b661-444a-a40f-011ae4f0911a, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External } } } }, OmicronZoneConfig { id: f64b0db4-4e9f-4666-b267-c9abe9f8b189, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 56d79ac6-bb86-49e7-905e-343e52dbbb90, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd06e234-5970-4365-a0e5-6cd29811daea (zpool), kind: External } } } }, OmicronZoneConfig { id: 010164b7-56ba-4dda-8f67-3d5e05751e2e, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3488ba32-8748-4793-9389-5ea882fa963c, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool), kind: External } } } }, OmicronZoneConfig { id: 89e132b1-d133-478e-a8f7-cf9f30b211c9, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:00.177083895Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:00.177460488Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:00.177472461Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9f65db5e-d949-4342-9d90-6d691adea120, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:00.204686519Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:00.204742061Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9f65db5e-d949-4342-9d90-6d691adea120, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:00.231532536Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:00.231686459Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e3552cf6-27c7-4100-942b-c5877c673046, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:00.256842876Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:00.256894997Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e3552cf6-27c7-4100-942b-c5877c673046, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a0125d1-b636-4039-9e54-dea22726a38a, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:00.282543846Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:00.282581497Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a0125d1-b636-4039-9e54-dea22726a38a, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c851202-bc92-4a32-b946-68379ad242d7, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:00.307937208Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:00.307972159Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7c851202-bc92-4a32-b946-68379ad242d7, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 653e8bac-3f5b-49fe-a9f9-e9f0546059e9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:00.333502115Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:00.333538845Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 653e8bac-3f5b-49fe-a9f9-e9f0546059e9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e4ffe9aa-395e-4536-a4d0-5cf0bc814584, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:00.357273139Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:00.35730782Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e4ffe9aa-395e-4536-a4d0-5cf0bc814584, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48ce280c-ebea-49f3-9829-dc9b4d018a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:00.381771437Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:00.381885483Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 48ce280c-ebea-49f3-9829-dc9b4d018a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:00.406997289Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:00.40703494Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 96bb9301-f50f-4259-8a1b-169ec98cbff3, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:00.432214999Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:00.432245112Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 96bb9301-f50f-4259-8a1b-169ec98cbff3, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c3d43ade-9569-403a-a6cc-494ec947793f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:00.456939684Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:00.456987205Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c3d43ade-9569-403a-a6cc-494ec947793f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f09c8d14-475d-49d8-b03f-d3a0b7b880cc, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:00.480246778Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:00.480300999Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f09c8d14-475d-49d8-b03f-d3a0b7b880cc, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b202e971-de3b-4a73-a534-ee56c23e79a3, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:00.506692105Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:00.506730336Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b202e971-de3b-4a73-a534-ee56c23e79a3, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b78106a-b661-444a-a40f-011ae4f0911a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:00.531457013Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:00.531496524Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2b78106a-b661-444a-a40f-011ae4f0911a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f64b0db4-4e9f-4666-b267-c9abe9f8b189, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:00.55577075Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:00.555804991Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f64b0db4-4e9f-4666-b267-c9abe9f8b189, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56d79ac6-bb86-49e7-905e-343e52dbbb90, dataset_name: DatasetName { pool_name: ZpoolName { id: bd06e234-5970-4365-a0e5-6cd29811daea (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:00.57764566Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:00.577677411Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 56d79ac6-bb86-49e7-905e-343e52dbbb90, dataset_name: DatasetName { pool_name: ZpoolName { id: bd06e234-5970-4365-a0e5-6cd29811daea (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 010164b7-56ba-4dda-8f67-3d5e05751e2e, dataset_name: DatasetName { pool_name: ZpoolName { id: e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:00.598410805Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:00.598449486Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 010164b7-56ba-4dda-8f67-3d5e05751e2e, dataset_name: DatasetName { pool_name: ZpoolName { id: e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3488ba32-8748-4793-9389-5ea882fa963c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:00.618945764Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:00.618976964Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3488ba32-8748-4793-9389-5ea882fa963c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89e132b1-d133-478e-a8f7-cf9f30b211c9, dataset_name: DatasetName { pool_name: ZpoolName { id: cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:00.639485993Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:00.639525834Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 89e132b1-d133-478e-a8f7-cf9f30b211c9, dataset_name: DatasetName { pool_name: ZpoolName { id: cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:00.660519144Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/968bbfe8-8122-4a85-96dd-954188a2f89d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:00.660551765Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/e8396e20-0493-4030-b55b-5c9e59a237cf/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:00.66078709Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:00.660943114Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:00.760455335Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_f09c8d14-475d-49d8-b03f-d3a0b7b880cc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:00.760510237Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:00.760517267Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:00.760544957Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:00.760553098Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:00.760558618Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_b202e971-de3b-4a73-a534-ee56c23e79a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:00.760565648Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:00.760571518Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:00.760576748Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:00.760583748Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:00.760590428Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:00.760595728Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:00.760602879Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:00.760606899Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:00.760610489Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:00.760615789Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:00.760619899Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:00.760625349Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:00.760633999Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:00.760691931Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"e56d899a-5c91-4be4-aaee-067aa7ed10a7","remote_addr":"[fd00:1122:3344:101::1]:39162","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_f09c8d14-475d-49d8-b03f-d3a0b7b880cc\", TimeNotSynchronized), (\"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717\", TimeNotSynchronized), (\"oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94\", TimeNotSynchronized), (\"oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c\", TimeNotSynchronized), (\"oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02\", TimeNotSynchronized), (\"oxz_crucible_b202e971-de3b-4a73-a534-ee56c23e79a3\", TimeNotSynchronized), (\"oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9\", TimeNotSynchronized), (\"oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90\", TimeNotSynchronized), (\"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93\", TimeNotSynchronized), (\"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075\", TimeNotSynchronized), (\"oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02\", TimeNotSynchronized), (\"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752\", TimeNotSynchronized), (\"oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189\", TimeNotSynchronized), (\"oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139\", TimeNotSynchronized), (\"oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4\", TimeNotSynchronized), (\"oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e\", TimeNotSynchronized), (\"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f\", TimeNotSynchronized), (\"oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a\", TimeNotSynchronized), (\"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3\", TimeNotSynchronized)]","latency_us":583367,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:00.760876005Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"324.841164ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e56d899a-5c91-4be4-aaee-067aa7ed10a7\", \"content-length\": \"124\", \"date\": \"Wed, 26 Jun 2024 05:36:00 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"e56d899a-5c91-4be4-aaee-067aa7ed10a7\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 9f65db5e-d949-4342-9d90-6d691adea120, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: a4d358ae-9d04-40a4-b4a5-ac8418ace88d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 707beb4d-acae-4ff1-bb7e-97992526c5fb, kind: Service { id: a4d358ae-9d04-40a4-b4a5-ac8418ace88d }, name: Name(\"ntp-a4d358ae-9d04-40a4-b4a5-ac8418ace88d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 238, 27])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e3552cf6-27c7-4100-942b-c5877c673046, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } } } }, OmicronZoneConfig { id: 9a0125d1-b636-4039-9e54-dea22726a38a, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } } } }, OmicronZoneConfig { id: 7c851202-bc92-4a32-b946-68379ad242d7, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 653e8bac-3f5b-49fe-a9f9-e9f0546059e9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External } } } }, OmicronZoneConfig { id: e4ffe9aa-395e-4536-a4d0-5cf0bc814584, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 48ce280c-ebea-49f3-9829-dc9b4d018a02, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: c2791662-4c87-4a63-a67a-4d6489fb4eaf, kind: Service { id: 48ce280c-ebea-49f3-9829-dc9b4d018a02 }, name: Name(\"external-dns-48ce280c-ebea-49f3-9829-dc9b4d018a02\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 86])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 38cec1c6-d097-4604-b9cb-30a5fab57e35, kind: Service { id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94 }, name: Name(\"external-dns-9f29eb53-b3d5-4fc1-b4d8-a252757e0c94\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 215])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 67179f00-d639-4f5b-b3c6-71d676113139, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: a724cf8c-c466-4981-aa80-a17ab2349f83, kind: Service { id: 67179f00-d639-4f5b-b3c6-71d676113139 }, name: Name(\"nexus-67179f00-d639-4f5b-b3c6-71d676113139\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 55])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7e2a6156-04b4-4797-8bed-d0c5c9cbe752, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: e31530f0-72e9-4544-bfbb-847c405d46c8, kind: Service { id: 7e2a6156-04b4-4797-8bed-d0c5c9cbe752 }, name: Name(\"nexus-7e2a6156-04b4-4797-8bed-d0c5c9cbe752\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 20a0479c-19b7-40a0-a935-5ff5399f7d02, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 79c95d4d-cff5-4419-8f3e-a6e216736b73, kind: Service { id: 20a0479c-19b7-40a0-a935-5ff5399f7d02 }, name: Name(\"nexus-20a0479c-19b7-40a0-a935-5ff5399f7d02\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 249])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: e42e8525-00d2-488f-a732-5c9514a5c7b4, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 96bb9301-f50f-4259-8a1b-169ec98cbff3, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } } } }, OmicronZoneConfig { id: 966bee63-3642-4e7a-a159-17eb59b72717, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: da8be60e-c115-4289-8526-a1ad7b366b93, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: d72762b4-6a43-4e1c-92b1-b432c1731075, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: c3d43ade-9569-403a-a6cc-494ec947793f, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } } } }, OmicronZoneConfig { id: f09c8d14-475d-49d8-b03f-d3a0b7b880cc, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } } } }, OmicronZoneConfig { id: b202e971-de3b-4a73-a534-ee56c23e79a3, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2b78106a-b661-444a-a40f-011ae4f0911a, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External } } } }, OmicronZoneConfig { id: f64b0db4-4e9f-4666-b267-c9abe9f8b189, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 56d79ac6-bb86-49e7-905e-343e52dbbb90, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd06e234-5970-4365-a0e5-6cd29811daea (zpool), kind: External } } } }, OmicronZoneConfig { id: 010164b7-56ba-4dda-8f67-3d5e05751e2e, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3488ba32-8748-4793-9389-5ea882fa963c, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool), kind: External } } } }, OmicronZoneConfig { id: 89e132b1-d133-478e-a8f7-cf9f30b211c9, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:01.086770738Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:01.087154517Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:01.087171307Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9f65db5e-d949-4342-9d90-6d691adea120, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:01.109390986Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:01.109441777Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9f65db5e-d949-4342-9d90-6d691adea120, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:01.128295687Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:01.128343068Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e3552cf6-27c7-4100-942b-c5877c673046, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:01.147211638Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:01.147267829Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e3552cf6-27c7-4100-942b-c5877c673046, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a0125d1-b636-4039-9e54-dea22726a38a, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:01.165675199Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:01.16571974Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a0125d1-b636-4039-9e54-dea22726a38a, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c851202-bc92-4a32-b946-68379ad242d7, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:01.185240485Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:01.185284716Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7c851202-bc92-4a32-b946-68379ad242d7, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 653e8bac-3f5b-49fe-a9f9-e9f0546059e9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:01.204451766Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:01.20449087Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 653e8bac-3f5b-49fe-a9f9-e9f0546059e9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e4ffe9aa-395e-4536-a4d0-5cf0bc814584, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:01.222895954Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:01.222943605Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e4ffe9aa-395e-4536-a4d0-5cf0bc814584, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48ce280c-ebea-49f3-9829-dc9b4d018a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:01.242964102Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:01.243013063Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 48ce280c-ebea-49f3-9829-dc9b4d018a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:01.26260997Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:01.262660412Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 96bb9301-f50f-4259-8a1b-169ec98cbff3, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:01.282185777Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:01.282232588Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 96bb9301-f50f-4259-8a1b-169ec98cbff3, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c3d43ade-9569-403a-a6cc-494ec947793f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:01.301721213Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:01.301761434Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c3d43ade-9569-403a-a6cc-494ec947793f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f09c8d14-475d-49d8-b03f-d3a0b7b880cc, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:01.32217939Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:01.322227731Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f09c8d14-475d-49d8-b03f-d3a0b7b880cc, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b202e971-de3b-4a73-a534-ee56c23e79a3, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:01.34188885Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:01.341930901Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b202e971-de3b-4a73-a534-ee56c23e79a3, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b78106a-b661-444a-a40f-011ae4f0911a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:01.360241018Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:01.360281849Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2b78106a-b661-444a-a40f-011ae4f0911a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f64b0db4-4e9f-4666-b267-c9abe9f8b189, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:01.379578659Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:01.37962314Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f64b0db4-4e9f-4666-b267-c9abe9f8b189, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56d79ac6-bb86-49e7-905e-343e52dbbb90, dataset_name: DatasetName { pool_name: ZpoolName { id: bd06e234-5970-4365-a0e5-6cd29811daea (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:01.398834408Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:01.398872771Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 56d79ac6-bb86-49e7-905e-343e52dbbb90, dataset_name: DatasetName { pool_name: ZpoolName { id: bd06e234-5970-4365-a0e5-6cd29811daea (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 010164b7-56ba-4dda-8f67-3d5e05751e2e, dataset_name: DatasetName { pool_name: ZpoolName { id: e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:01.419841048Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:01.41989877Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 010164b7-56ba-4dda-8f67-3d5e05751e2e, dataset_name: DatasetName { pool_name: ZpoolName { id: e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3488ba32-8748-4793-9389-5ea882fa963c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:01.438967394Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:01.439013215Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3488ba32-8748-4793-9389-5ea882fa963c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89e132b1-d133-478e-a8f7-cf9f30b211c9, dataset_name: DatasetName { pool_name: ZpoolName { id: cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:01.45957892Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:01.459746559Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 89e132b1-d133-478e-a8f7-cf9f30b211c9, dataset_name: DatasetName { pool_name: ZpoolName { id: cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:01.477237607Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/968bbfe8-8122-4a85-96dd-954188a2f89d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:01.477292379Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/e8396e20-0493-4030-b55b-5c9e59a237cf/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:01.477490253Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:01.477611276Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:01.574148278Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:01.5741995Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:01.57421603Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:01.5742226Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:01.57422679Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_b202e971-de3b-4a73-a534-ee56c23e79a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:01.57423111Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:01.57423627Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:01.57424082Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:01.57424597Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:01.574250261Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:01.574254481Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:01.574259681Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_f09c8d14-475d-49d8-b03f-d3a0b7b880cc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:01.574263921Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:01.574269041Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:01.574273271Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:01.574278471Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:01.574289271Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:01.574305882Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:01.574321792Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:01.574439035Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"2a922b13-886b-4e23-b813-e2ddd5f61472","remote_addr":"[fd00:1122:3344:101::1]:39162","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3\", TimeNotSynchronized), (\"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717\", TimeNotSynchronized), (\"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075\", TimeNotSynchronized), (\"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f\", TimeNotSynchronized), (\"oxz_crucible_b202e971-de3b-4a73-a534-ee56c23e79a3\", TimeNotSynchronized), (\"oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a\", TimeNotSynchronized), (\"oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189\", TimeNotSynchronized), (\"oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c\", TimeNotSynchronized), (\"oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e\", TimeNotSynchronized), (\"oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139\", TimeNotSynchronized), (\"oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02\", TimeNotSynchronized), (\"oxz_crucible_f09c8d14-475d-49d8-b03f-d3a0b7b880cc\", TimeNotSynchronized), (\"oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02\", TimeNotSynchronized), (\"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93\", TimeNotSynchronized), (\"oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90\", TimeNotSynchronized), (\"oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9\", TimeNotSynchronized), (\"oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4\", TimeNotSynchronized), (\"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752\", TimeNotSynchronized), (\"oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94\", TimeNotSynchronized)]","latency_us":487480,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:01.574609849Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"341.020444ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"2a922b13-886b-4e23-b813-e2ddd5f61472\", \"content-length\": \"124\", \"date\": \"Wed, 26 Jun 2024 05:36:01 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"2a922b13-886b-4e23-b813-e2ddd5f61472\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 9f65db5e-d949-4342-9d90-6d691adea120, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: a4d358ae-9d04-40a4-b4a5-ac8418ace88d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 707beb4d-acae-4ff1-bb7e-97992526c5fb, kind: Service { id: a4d358ae-9d04-40a4-b4a5-ac8418ace88d }, name: Name(\"ntp-a4d358ae-9d04-40a4-b4a5-ac8418ace88d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 238, 27])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e3552cf6-27c7-4100-942b-c5877c673046, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } } } }, OmicronZoneConfig { id: 9a0125d1-b636-4039-9e54-dea22726a38a, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } } } }, OmicronZoneConfig { id: 7c851202-bc92-4a32-b946-68379ad242d7, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 653e8bac-3f5b-49fe-a9f9-e9f0546059e9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External } } } }, OmicronZoneConfig { id: e4ffe9aa-395e-4536-a4d0-5cf0bc814584, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 48ce280c-ebea-49f3-9829-dc9b4d018a02, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: c2791662-4c87-4a63-a67a-4d6489fb4eaf, kind: Service { id: 48ce280c-ebea-49f3-9829-dc9b4d018a02 }, name: Name(\"external-dns-48ce280c-ebea-49f3-9829-dc9b4d018a02\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 86])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 38cec1c6-d097-4604-b9cb-30a5fab57e35, kind: Service { id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94 }, name: Name(\"external-dns-9f29eb53-b3d5-4fc1-b4d8-a252757e0c94\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 215])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 67179f00-d639-4f5b-b3c6-71d676113139, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: a724cf8c-c466-4981-aa80-a17ab2349f83, kind: Service { id: 67179f00-d639-4f5b-b3c6-71d676113139 }, name: Name(\"nexus-67179f00-d639-4f5b-b3c6-71d676113139\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 55])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7e2a6156-04b4-4797-8bed-d0c5c9cbe752, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: e31530f0-72e9-4544-bfbb-847c405d46c8, kind: Service { id: 7e2a6156-04b4-4797-8bed-d0c5c9cbe752 }, name: Name(\"nexus-7e2a6156-04b4-4797-8bed-d0c5c9cbe752\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 20a0479c-19b7-40a0-a935-5ff5399f7d02, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 79c95d4d-cff5-4419-8f3e-a6e216736b73, kind: Service { id: 20a0479c-19b7-40a0-a935-5ff5399f7d02 }, name: Name(\"nexus-20a0479c-19b7-40a0-a935-5ff5399f7d02\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 249])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: e42e8525-00d2-488f-a732-5c9514a5c7b4, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 96bb9301-f50f-4259-8a1b-169ec98cbff3, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } } } }, OmicronZoneConfig { id: 966bee63-3642-4e7a-a159-17eb59b72717, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: da8be60e-c115-4289-8526-a1ad7b366b93, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: d72762b4-6a43-4e1c-92b1-b432c1731075, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: c3d43ade-9569-403a-a6cc-494ec947793f, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } } } }, OmicronZoneConfig { id: f09c8d14-475d-49d8-b03f-d3a0b7b880cc, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } } } }, OmicronZoneConfig { id: b202e971-de3b-4a73-a534-ee56c23e79a3, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2b78106a-b661-444a-a40f-011ae4f0911a, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External } } } }, OmicronZoneConfig { id: f64b0db4-4e9f-4666-b267-c9abe9f8b189, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 56d79ac6-bb86-49e7-905e-343e52dbbb90, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd06e234-5970-4365-a0e5-6cd29811daea (zpool), kind: External } } } }, OmicronZoneConfig { id: 010164b7-56ba-4dda-8f67-3d5e05751e2e, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3488ba32-8748-4793-9389-5ea882fa963c, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool), kind: External } } } }, OmicronZoneConfig { id: 89e132b1-d133-478e-a8f7-cf9f30b211c9, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:01.916740939Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:01.917292774Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:01.917301074Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9f65db5e-d949-4342-9d90-6d691adea120, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:01.939681866Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:01.939716587Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9f65db5e-d949-4342-9d90-6d691adea120, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:01.960772608Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:01.960807369Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e3552cf6-27c7-4100-942b-c5877c673046, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:01.982335351Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:01.982381282Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e3552cf6-27c7-4100-942b-c5877c673046, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a0125d1-b636-4039-9e54-dea22726a38a, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:02.001388696Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:02.001420626Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a0125d1-b636-4039-9e54-dea22726a38a, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c851202-bc92-4a32-b946-68379ad242d7, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:02.022962639Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:02.02299659Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7c851202-bc92-4a32-b946-68379ad242d7, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 653e8bac-3f5b-49fe-a9f9-e9f0546059e9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:02.044083562Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:02.044118163Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 653e8bac-3f5b-49fe-a9f9-e9f0546059e9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e4ffe9aa-395e-4536-a4d0-5cf0bc814584, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:02.063191177Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:02.063225468Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e4ffe9aa-395e-4536-a4d0-5cf0bc814584, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48ce280c-ebea-49f3-9829-dc9b4d018a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:02.081537236Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:02.081570206Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 48ce280c-ebea-49f3-9829-dc9b4d018a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:02.100356164Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:02.100522068Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 96bb9301-f50f-4259-8a1b-169ec98cbff3, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:02.119144173Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:02.119175664Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 96bb9301-f50f-4259-8a1b-169ec98cbff3, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c3d43ade-9569-403a-a6cc-494ec947793f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:02.137480701Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:02.137514802Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c3d43ade-9569-403a-a6cc-494ec947793f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f09c8d14-475d-49d8-b03f-d3a0b7b880cc, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:02.156096925Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:02.156139146Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f09c8d14-475d-49d8-b03f-d3a0b7b880cc, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b202e971-de3b-4a73-a534-ee56c23e79a3, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:02.174488614Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:02.174518105Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b202e971-de3b-4a73-a534-ee56c23e79a3, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b78106a-b661-444a-a40f-011ae4f0911a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:02.19276746Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:02.192801491Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2b78106a-b661-444a-a40f-011ae4f0911a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f64b0db4-4e9f-4666-b267-c9abe9f8b189, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:02.210877623Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:02.210906224Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f64b0db4-4e9f-4666-b267-c9abe9f8b189, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56d79ac6-bb86-49e7-905e-343e52dbbb90, dataset_name: DatasetName { pool_name: ZpoolName { id: bd06e234-5970-4365-a0e5-6cd29811daea (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:02.229844285Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:02.229876146Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 56d79ac6-bb86-49e7-905e-343e52dbbb90, dataset_name: DatasetName { pool_name: ZpoolName { id: bd06e234-5970-4365-a0e5-6cd29811daea (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 010164b7-56ba-4dda-8f67-3d5e05751e2e, dataset_name: DatasetName { pool_name: ZpoolName { id: e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:02.248799548Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:02.248829278Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 010164b7-56ba-4dda-8f67-3d5e05751e2e, dataset_name: DatasetName { pool_name: ZpoolName { id: e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3488ba32-8748-4793-9389-5ea882fa963c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:02.268881706Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:02.268920627Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3488ba32-8748-4793-9389-5ea882fa963c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89e132b1-d133-478e-a8f7-cf9f30b211c9, dataset_name: DatasetName { pool_name: ZpoolName { id: cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:02.287819608Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:02.287858919Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 89e132b1-d133-478e-a8f7-cf9f30b211c9, dataset_name: DatasetName { pool_name: ZpoolName { id: cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:02.306559575Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/968bbfe8-8122-4a85-96dd-954188a2f89d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:02.306600256Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/e8396e20-0493-4030-b55b-5c9e59a237cf/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:02.306827221Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:02.306957535Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:02.404538941Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:02.404586642Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:02.404607296Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:02.404611561Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:02.404615096Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:02.404619052Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:02.404622755Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:02.404626196Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:02.404629796Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:02.404633293Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:02.40463679Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:02.404640493Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:02.40464399Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:02.40464773Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:02.404701826Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:02.404708632Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:02.404712673Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_f09c8d14-475d-49d8-b03f-d3a0b7b880cc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:02.404717679Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_b202e971-de3b-4a73-a534-ee56c23e79a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:02.40472592Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:02.404872145Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"edbe83ef-7be9-4b89-b949-87956d79f6ec","remote_addr":"[fd00:1122:3344:101::1]:39162","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189\", TimeNotSynchronized), (\"oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139\", TimeNotSynchronized), (\"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f\", TimeNotSynchronized), (\"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752\", TimeNotSynchronized), (\"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93\", TimeNotSynchronized), (\"oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02\", TimeNotSynchronized), (\"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3\", TimeNotSynchronized), (\"oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4\", TimeNotSynchronized), (\"oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a\", TimeNotSynchronized), (\"oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9\", TimeNotSynchronized), (\"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075\", TimeNotSynchronized), (\"oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90\", TimeNotSynchronized), (\"oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e\", TimeNotSynchronized), (\"oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c\", TimeNotSynchronized), (\"oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94\", TimeNotSynchronized), (\"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717\", TimeNotSynchronized), (\"oxz_crucible_f09c8d14-475d-49d8-b03f-d3a0b7b880cc\", TimeNotSynchronized), (\"oxz_crucible_b202e971-de3b-4a73-a534-ee56c23e79a3\", TimeNotSynchronized), (\"oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02\", TimeNotSynchronized)]","latency_us":487908,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:02.405127435Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"478.016412ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"edbe83ef-7be9-4b89-b949-87956d79f6ec\", \"content-length\": \"124\", \"date\": \"Wed, 26 Jun 2024 05:36:01 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"edbe83ef-7be9-4b89-b949-87956d79f6ec\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 9f65db5e-d949-4342-9d90-6d691adea120, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: a4d358ae-9d04-40a4-b4a5-ac8418ace88d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 707beb4d-acae-4ff1-bb7e-97992526c5fb, kind: Service { id: a4d358ae-9d04-40a4-b4a5-ac8418ace88d }, name: Name(\"ntp-a4d358ae-9d04-40a4-b4a5-ac8418ace88d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 238, 27])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e3552cf6-27c7-4100-942b-c5877c673046, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } } } }, OmicronZoneConfig { id: 9a0125d1-b636-4039-9e54-dea22726a38a, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } } } }, OmicronZoneConfig { id: 7c851202-bc92-4a32-b946-68379ad242d7, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 653e8bac-3f5b-49fe-a9f9-e9f0546059e9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External } } } }, OmicronZoneConfig { id: e4ffe9aa-395e-4536-a4d0-5cf0bc814584, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 48ce280c-ebea-49f3-9829-dc9b4d018a02, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: c2791662-4c87-4a63-a67a-4d6489fb4eaf, kind: Service { id: 48ce280c-ebea-49f3-9829-dc9b4d018a02 }, name: Name(\"external-dns-48ce280c-ebea-49f3-9829-dc9b4d018a02\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 86])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 38cec1c6-d097-4604-b9cb-30a5fab57e35, kind: Service { id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94 }, name: Name(\"external-dns-9f29eb53-b3d5-4fc1-b4d8-a252757e0c94\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 215])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 67179f00-d639-4f5b-b3c6-71d676113139, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: a724cf8c-c466-4981-aa80-a17ab2349f83, kind: Service { id: 67179f00-d639-4f5b-b3c6-71d676113139 }, name: Name(\"nexus-67179f00-d639-4f5b-b3c6-71d676113139\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 55])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7e2a6156-04b4-4797-8bed-d0c5c9cbe752, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: e31530f0-72e9-4544-bfbb-847c405d46c8, kind: Service { id: 7e2a6156-04b4-4797-8bed-d0c5c9cbe752 }, name: Name(\"nexus-7e2a6156-04b4-4797-8bed-d0c5c9cbe752\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 20a0479c-19b7-40a0-a935-5ff5399f7d02, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 79c95d4d-cff5-4419-8f3e-a6e216736b73, kind: Service { id: 20a0479c-19b7-40a0-a935-5ff5399f7d02 }, name: Name(\"nexus-20a0479c-19b7-40a0-a935-5ff5399f7d02\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 249])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: e42e8525-00d2-488f-a732-5c9514a5c7b4, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 96bb9301-f50f-4259-8a1b-169ec98cbff3, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } } } }, OmicronZoneConfig { id: 966bee63-3642-4e7a-a159-17eb59b72717, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: da8be60e-c115-4289-8526-a1ad7b366b93, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: d72762b4-6a43-4e1c-92b1-b432c1731075, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: c3d43ade-9569-403a-a6cc-494ec947793f, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } } } }, OmicronZoneConfig { id: f09c8d14-475d-49d8-b03f-d3a0b7b880cc, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } } } }, OmicronZoneConfig { id: b202e971-de3b-4a73-a534-ee56c23e79a3, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2b78106a-b661-444a-a40f-011ae4f0911a, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External } } } }, OmicronZoneConfig { id: f64b0db4-4e9f-4666-b267-c9abe9f8b189, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 56d79ac6-bb86-49e7-905e-343e52dbbb90, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd06e234-5970-4365-a0e5-6cd29811daea (zpool), kind: External } } } }, OmicronZoneConfig { id: 010164b7-56ba-4dda-8f67-3d5e05751e2e, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3488ba32-8748-4793-9389-5ea882fa963c, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool), kind: External } } } }, OmicronZoneConfig { id: 89e132b1-d133-478e-a8f7-cf9f30b211c9, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:02.883462564Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:02.883791772Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:02.883801502Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9f65db5e-d949-4342-9d90-6d691adea120, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:02.904844553Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:02.904893444Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9f65db5e-d949-4342-9d90-6d691adea120, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:02.925519795Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:02.925558356Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e3552cf6-27c7-4100-942b-c5877c673046, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:02.943468322Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:02.943507885Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e3552cf6-27c7-4100-942b-c5877c673046, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a0125d1-b636-4039-9e54-dea22726a38a, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:02.963307627Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:02.963344788Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a0125d1-b636-4039-9e54-dea22726a38a, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c851202-bc92-4a32-b946-68379ad242d7, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:02.980111589Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:02.98014819Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7c851202-bc92-4a32-b946-68379ad242d7, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 653e8bac-3f5b-49fe-a9f9-e9f0546059e9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:02.997900694Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:02.997938735Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 653e8bac-3f5b-49fe-a9f9-e9f0546059e9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e4ffe9aa-395e-4536-a4d0-5cf0bc814584, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:03.01487904Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:03.01493696Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e4ffe9aa-395e-4536-a4d0-5cf0bc814584, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48ce280c-ebea-49f3-9829-dc9b4d018a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:03.033828632Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:03.033878893Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 48ce280c-ebea-49f3-9829-dc9b4d018a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:03.054495544Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:03.054538875Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 96bb9301-f50f-4259-8a1b-169ec98cbff3, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:03.075601941Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:03.075637416Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 96bb9301-f50f-4259-8a1b-169ec98cbff3, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c3d43ade-9569-403a-a6cc-494ec947793f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:03.094559219Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:03.0946002Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c3d43ade-9569-403a-a6cc-494ec947793f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f09c8d14-475d-49d8-b03f-d3a0b7b880cc, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:03.112700492Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:03.112742753Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f09c8d14-475d-49d8-b03f-d3a0b7b880cc, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b202e971-de3b-4a73-a534-ee56c23e79a3, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:03.131507491Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:03.131547392Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b202e971-de3b-4a73-a534-ee56c23e79a3, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b78106a-b661-444a-a40f-011ae4f0911a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:03.151764904Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:03.151905467Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2b78106a-b661-444a-a40f-011ae4f0911a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f64b0db4-4e9f-4666-b267-c9abe9f8b189, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:03.171725179Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:03.17176245Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f64b0db4-4e9f-4666-b267-c9abe9f8b189, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56d79ac6-bb86-49e7-905e-343e52dbbb90, dataset_name: DatasetName { pool_name: ZpoolName { id: bd06e234-5970-4365-a0e5-6cd29811daea (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:03.193312843Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:03.193352984Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 56d79ac6-bb86-49e7-905e-343e52dbbb90, dataset_name: DatasetName { pool_name: ZpoolName { id: bd06e234-5970-4365-a0e5-6cd29811daea (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 010164b7-56ba-4dda-8f67-3d5e05751e2e, dataset_name: DatasetName { pool_name: ZpoolName { id: e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:03.212791257Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:03.212827438Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 010164b7-56ba-4dda-8f67-3d5e05751e2e, dataset_name: DatasetName { pool_name: ZpoolName { id: e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3488ba32-8748-4793-9389-5ea882fa963c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:03.230812898Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:03.230851049Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3488ba32-8748-4793-9389-5ea882fa963c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89e132b1-d133-478e-a8f7-cf9f30b211c9, dataset_name: DatasetName { pool_name: ZpoolName { id: cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:03.250898586Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:03.250941077Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 89e132b1-d133-478e-a8f7-cf9f30b211c9, dataset_name: DatasetName { pool_name: ZpoolName { id: cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:03.268819775Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/968bbfe8-8122-4a85-96dd-954188a2f89d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:03.268875716Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/e8396e20-0493-4030-b55b-5c9e59a237cf/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:03.269088201Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:03.26920658Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:03.364987127Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:03.365033218Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:03.365048109Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_f09c8d14-475d-49d8-b03f-d3a0b7b880cc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:03.365053669Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:03.365059579Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:03.365063809Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:03.365068929Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:03.365073239Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:03.365077779Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:03.3650828Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:03.365086229Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:03.36509127Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_b202e971-de3b-4a73-a534-ee56c23e79a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:03.3650955Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:03.36510066Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:03.36510384Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:03.36510799Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:03.36511335Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:03.365128421Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:03.365133221Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:03.365244363Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"7edc02e4-70d6-4017-be1b-2f513262e567","remote_addr":"[fd00:1122:3344:101::1]:39162","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752\", TimeNotSynchronized), (\"oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90\", TimeNotSynchronized), (\"oxz_crucible_f09c8d14-475d-49d8-b03f-d3a0b7b880cc\", TimeNotSynchronized), (\"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075\", TimeNotSynchronized), (\"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93\", TimeNotSynchronized), (\"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f\", TimeNotSynchronized), (\"oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9\", TimeNotSynchronized), (\"oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02\", TimeNotSynchronized), (\"oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94\", TimeNotSynchronized), (\"oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4\", TimeNotSynchronized), (\"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717\", TimeNotSynchronized), (\"oxz_crucible_b202e971-de3b-4a73-a534-ee56c23e79a3\", TimeNotSynchronized), (\"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3\", TimeNotSynchronized), (\"oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a\", TimeNotSynchronized), (\"oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02\", TimeNotSynchronized), (\"oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189\", TimeNotSynchronized), (\"oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e\", TimeNotSynchronized), (\"oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139\", TimeNotSynchronized), (\"oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c\", TimeNotSynchronized)]","latency_us":481594,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:03.365435948Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"463.479442ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"7edc02e4-70d6-4017-be1b-2f513262e567\", \"content-length\": \"124\", \"date\": \"Wed, 26 Jun 2024 05:36:02 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"7edc02e4-70d6-4017-be1b-2f513262e567\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 9f65db5e-d949-4342-9d90-6d691adea120, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: a4d358ae-9d04-40a4-b4a5-ac8418ace88d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 707beb4d-acae-4ff1-bb7e-97992526c5fb, kind: Service { id: a4d358ae-9d04-40a4-b4a5-ac8418ace88d }, name: Name(\"ntp-a4d358ae-9d04-40a4-b4a5-ac8418ace88d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 238, 27])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e3552cf6-27c7-4100-942b-c5877c673046, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } } } }, OmicronZoneConfig { id: 9a0125d1-b636-4039-9e54-dea22726a38a, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } } } }, OmicronZoneConfig { id: 7c851202-bc92-4a32-b946-68379ad242d7, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 653e8bac-3f5b-49fe-a9f9-e9f0546059e9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External } } } }, OmicronZoneConfig { id: e4ffe9aa-395e-4536-a4d0-5cf0bc814584, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 48ce280c-ebea-49f3-9829-dc9b4d018a02, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: c2791662-4c87-4a63-a67a-4d6489fb4eaf, kind: Service { id: 48ce280c-ebea-49f3-9829-dc9b4d018a02 }, name: Name(\"external-dns-48ce280c-ebea-49f3-9829-dc9b4d018a02\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 86])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 38cec1c6-d097-4604-b9cb-30a5fab57e35, kind: Service { id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94 }, name: Name(\"external-dns-9f29eb53-b3d5-4fc1-b4d8-a252757e0c94\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 215])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 67179f00-d639-4f5b-b3c6-71d676113139, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: a724cf8c-c466-4981-aa80-a17ab2349f83, kind: Service { id: 67179f00-d639-4f5b-b3c6-71d676113139 }, name: Name(\"nexus-67179f00-d639-4f5b-b3c6-71d676113139\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 55])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7e2a6156-04b4-4797-8bed-d0c5c9cbe752, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: e31530f0-72e9-4544-bfbb-847c405d46c8, kind: Service { id: 7e2a6156-04b4-4797-8bed-d0c5c9cbe752 }, name: Name(\"nexus-7e2a6156-04b4-4797-8bed-d0c5c9cbe752\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 20a0479c-19b7-40a0-a935-5ff5399f7d02, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 79c95d4d-cff5-4419-8f3e-a6e216736b73, kind: Service { id: 20a0479c-19b7-40a0-a935-5ff5399f7d02 }, name: Name(\"nexus-20a0479c-19b7-40a0-a935-5ff5399f7d02\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 249])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: e42e8525-00d2-488f-a732-5c9514a5c7b4, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 96bb9301-f50f-4259-8a1b-169ec98cbff3, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } } } }, OmicronZoneConfig { id: 966bee63-3642-4e7a-a159-17eb59b72717, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: da8be60e-c115-4289-8526-a1ad7b366b93, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: d72762b4-6a43-4e1c-92b1-b432c1731075, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: c3d43ade-9569-403a-a6cc-494ec947793f, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } } } }, OmicronZoneConfig { id: f09c8d14-475d-49d8-b03f-d3a0b7b880cc, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } } } }, OmicronZoneConfig { id: b202e971-de3b-4a73-a534-ee56c23e79a3, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2b78106a-b661-444a-a40f-011ae4f0911a, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External } } } }, OmicronZoneConfig { id: f64b0db4-4e9f-4666-b267-c9abe9f8b189, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 56d79ac6-bb86-49e7-905e-343e52dbbb90, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd06e234-5970-4365-a0e5-6cd29811daea (zpool), kind: External } } } }, OmicronZoneConfig { id: 010164b7-56ba-4dda-8f67-3d5e05751e2e, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3488ba32-8748-4793-9389-5ea882fa963c, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool), kind: External } } } }, OmicronZoneConfig { id: 89e132b1-d133-478e-a8f7-cf9f30b211c9, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:03.83019576Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:03.830547449Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:03.830560019Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9f65db5e-d949-4342-9d90-6d691adea120, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:03.852973352Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:03.853005763Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9f65db5e-d949-4342-9d90-6d691adea120, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:03.874081884Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:03.874120855Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e3552cf6-27c7-4100-942b-c5877c673046, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:03.894249065Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:03.894282255Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e3552cf6-27c7-4100-942b-c5877c673046, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a0125d1-b636-4039-9e54-dea22726a38a, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:03.913061904Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:03.913102185Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a0125d1-b636-4039-9e54-dea22726a38a, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c851202-bc92-4a32-b946-68379ad242d7, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:03.931891433Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:03.931925524Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7c851202-bc92-4a32-b946-68379ad242d7, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 653e8bac-3f5b-49fe-a9f9-e9f0546059e9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:03.951388278Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:03.951422799Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 653e8bac-3f5b-49fe-a9f9-e9f0546059e9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e4ffe9aa-395e-4536-a4d0-5cf0bc814584, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:03.969720495Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:03.969765096Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e4ffe9aa-395e-4536-a4d0-5cf0bc814584, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48ce280c-ebea-49f3-9829-dc9b4d018a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:03.987305536Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:03.987340897Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 48ce280c-ebea-49f3-9829-dc9b4d018a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:04.005486246Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:04.00566056Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 96bb9301-f50f-4259-8a1b-169ec98cbff3, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:04.028643067Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:04.028675298Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 96bb9301-f50f-4259-8a1b-169ec98cbff3, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c3d43ade-9569-403a-a6cc-494ec947793f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:04.047522139Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:04.047555928Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c3d43ade-9569-403a-a6cc-494ec947793f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f09c8d14-475d-49d8-b03f-d3a0b7b880cc, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:04.06693791Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:04.066983998Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f09c8d14-475d-49d8-b03f-d3a0b7b880cc, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b202e971-de3b-4a73-a534-ee56c23e79a3, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:04.086861472Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:04.086894698Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b202e971-de3b-4a73-a534-ee56c23e79a3, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b78106a-b661-444a-a40f-011ae4f0911a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:04.106072762Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:04.1061022Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2b78106a-b661-444a-a40f-011ae4f0911a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f64b0db4-4e9f-4666-b267-c9abe9f8b189, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:04.125663752Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:04.125696678Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f64b0db4-4e9f-4666-b267-c9abe9f8b189, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56d79ac6-bb86-49e7-905e-343e52dbbb90, dataset_name: DatasetName { pool_name: ZpoolName { id: bd06e234-5970-4365-a0e5-6cd29811daea (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:04.144507864Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:04.144538343Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 56d79ac6-bb86-49e7-905e-343e52dbbb90, dataset_name: DatasetName { pool_name: ZpoolName { id: bd06e234-5970-4365-a0e5-6cd29811daea (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 010164b7-56ba-4dda-8f67-3d5e05751e2e, dataset_name: DatasetName { pool_name: ZpoolName { id: e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:04.164575872Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:04.164608095Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 010164b7-56ba-4dda-8f67-3d5e05751e2e, dataset_name: DatasetName { pool_name: ZpoolName { id: e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3488ba32-8748-4793-9389-5ea882fa963c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:04.183866994Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:04.183897858Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3488ba32-8748-4793-9389-5ea882fa963c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89e132b1-d133-478e-a8f7-cf9f30b211c9, dataset_name: DatasetName { pool_name: ZpoolName { id: cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:04.202921077Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:04.202955409Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 89e132b1-d133-478e-a8f7-cf9f30b211c9, dataset_name: DatasetName { pool_name: ZpoolName { id: cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:04.221286237Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/968bbfe8-8122-4a85-96dd-954188a2f89d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:04.221324319Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/e8396e20-0493-4030-b55b-5c9e59a237cf/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:04.221524049Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:04.221651683Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:04.317360541Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:04.317410793Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:04.317435758Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_b202e971-de3b-4a73-a534-ee56c23e79a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:04.317443052Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:04.317448246Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:04.317453008Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_f09c8d14-475d-49d8-b03f-d3a0b7b880cc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:04.317458062Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:04.31746274Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:04.31746934Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:04.317474581Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:04.317480919Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:04.317485897Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:04.317492159Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:04.317497616Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:04.31750116Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:04.317506082Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:04.317509729Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:04.317517923Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:04.317522882Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:04.317589624Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"f8b65e0d-be86-4fb6-ae7d-9022b941dc74","remote_addr":"[fd00:1122:3344:101::1]:39162","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9\", TimeNotSynchronized), (\"oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94\", TimeNotSynchronized), (\"oxz_crucible_b202e971-de3b-4a73-a534-ee56c23e79a3\", TimeNotSynchronized), (\"oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90\", TimeNotSynchronized), (\"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3\", TimeNotSynchronized), (\"oxz_crucible_f09c8d14-475d-49d8-b03f-d3a0b7b880cc\", TimeNotSynchronized), (\"oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02\", TimeNotSynchronized), (\"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075\", TimeNotSynchronized), (\"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717\", TimeNotSynchronized), (\"oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c\", TimeNotSynchronized), (\"oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02\", TimeNotSynchronized), (\"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752\", TimeNotSynchronized), (\"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f\", TimeNotSynchronized), (\"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93\", TimeNotSynchronized), (\"oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4\", TimeNotSynchronized), (\"oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189\", TimeNotSynchronized), (\"oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139\", TimeNotSynchronized), (\"oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a\", TimeNotSynchronized), (\"oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e\", TimeNotSynchronized)]","latency_us":486151,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:04.317766525Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"632.045539ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f8b65e0d-be86-4fb6-ae7d-9022b941dc74\", \"content-length\": \"124\", \"date\": \"Wed, 26 Jun 2024 05:36:03 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f8b65e0d-be86-4fb6-ae7d-9022b941dc74\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 9f65db5e-d949-4342-9d90-6d691adea120, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: a4d358ae-9d04-40a4-b4a5-ac8418ace88d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 707beb4d-acae-4ff1-bb7e-97992526c5fb, kind: Service { id: a4d358ae-9d04-40a4-b4a5-ac8418ace88d }, name: Name(\"ntp-a4d358ae-9d04-40a4-b4a5-ac8418ace88d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 238, 27])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e3552cf6-27c7-4100-942b-c5877c673046, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } } } }, OmicronZoneConfig { id: 9a0125d1-b636-4039-9e54-dea22726a38a, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } } } }, OmicronZoneConfig { id: 7c851202-bc92-4a32-b946-68379ad242d7, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 653e8bac-3f5b-49fe-a9f9-e9f0546059e9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External } } } }, OmicronZoneConfig { id: e4ffe9aa-395e-4536-a4d0-5cf0bc814584, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 48ce280c-ebea-49f3-9829-dc9b4d018a02, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: c2791662-4c87-4a63-a67a-4d6489fb4eaf, kind: Service { id: 48ce280c-ebea-49f3-9829-dc9b4d018a02 }, name: Name(\"external-dns-48ce280c-ebea-49f3-9829-dc9b4d018a02\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 86])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 38cec1c6-d097-4604-b9cb-30a5fab57e35, kind: Service { id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94 }, name: Name(\"external-dns-9f29eb53-b3d5-4fc1-b4d8-a252757e0c94\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 215])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 67179f00-d639-4f5b-b3c6-71d676113139, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: a724cf8c-c466-4981-aa80-a17ab2349f83, kind: Service { id: 67179f00-d639-4f5b-b3c6-71d676113139 }, name: Name(\"nexus-67179f00-d639-4f5b-b3c6-71d676113139\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 55])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7e2a6156-04b4-4797-8bed-d0c5c9cbe752, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: e31530f0-72e9-4544-bfbb-847c405d46c8, kind: Service { id: 7e2a6156-04b4-4797-8bed-d0c5c9cbe752 }, name: Name(\"nexus-7e2a6156-04b4-4797-8bed-d0c5c9cbe752\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 20a0479c-19b7-40a0-a935-5ff5399f7d02, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 79c95d4d-cff5-4419-8f3e-a6e216736b73, kind: Service { id: 20a0479c-19b7-40a0-a935-5ff5399f7d02 }, name: Name(\"nexus-20a0479c-19b7-40a0-a935-5ff5399f7d02\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 249])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: e42e8525-00d2-488f-a732-5c9514a5c7b4, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 96bb9301-f50f-4259-8a1b-169ec98cbff3, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } } } }, OmicronZoneConfig { id: 966bee63-3642-4e7a-a159-17eb59b72717, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: da8be60e-c115-4289-8526-a1ad7b366b93, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: d72762b4-6a43-4e1c-92b1-b432c1731075, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: c3d43ade-9569-403a-a6cc-494ec947793f, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } } } }, OmicronZoneConfig { id: f09c8d14-475d-49d8-b03f-d3a0b7b880cc, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } } } }, OmicronZoneConfig { id: b202e971-de3b-4a73-a534-ee56c23e79a3, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2b78106a-b661-444a-a40f-011ae4f0911a, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External } } } }, OmicronZoneConfig { id: f64b0db4-4e9f-4666-b267-c9abe9f8b189, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 56d79ac6-bb86-49e7-905e-343e52dbbb90, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd06e234-5970-4365-a0e5-6cd29811daea (zpool), kind: External } } } }, OmicronZoneConfig { id: 010164b7-56ba-4dda-8f67-3d5e05751e2e, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3488ba32-8748-4793-9389-5ea882fa963c, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool), kind: External } } } }, OmicronZoneConfig { id: 89e132b1-d133-478e-a8f7-cf9f30b211c9, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:04.953565049Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:04.953924972Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:04.953932745Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9f65db5e-d949-4342-9d90-6d691adea120, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:04.976350298Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:04.976400727Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9f65db5e-d949-4342-9d90-6d691adea120, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:04.99454244Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:04.994651027Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e3552cf6-27c7-4100-942b-c5877c673046, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:05.015604743Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:05.015717477Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e3552cf6-27c7-4100-942b-c5877c673046, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a0125d1-b636-4039-9e54-dea22726a38a, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:05.032469101Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:05.032511796Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a0125d1-b636-4039-9e54-dea22726a38a, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c851202-bc92-4a32-b946-68379ad242d7, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:05.048228253Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:05.048269251Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7c851202-bc92-4a32-b946-68379ad242d7, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 653e8bac-3f5b-49fe-a9f9-e9f0546059e9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:05.068287412Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:05.068338564Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 653e8bac-3f5b-49fe-a9f9-e9f0546059e9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e4ffe9aa-395e-4536-a4d0-5cf0bc814584, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:05.085491136Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:05.085546562Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e4ffe9aa-395e-4536-a4d0-5cf0bc814584, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48ce280c-ebea-49f3-9829-dc9b4d018a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:05.105943138Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:05.106051871Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 48ce280c-ebea-49f3-9829-dc9b4d018a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:05.125542018Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:05.12558985Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 96bb9301-f50f-4259-8a1b-169ec98cbff3, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:05.146581366Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:05.146620217Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 96bb9301-f50f-4259-8a1b-169ec98cbff3, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c3d43ade-9569-403a-a6cc-494ec947793f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:05.165675996Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:05.165716028Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c3d43ade-9569-403a-a6cc-494ec947793f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f09c8d14-475d-49d8-b03f-d3a0b7b880cc, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:05.184583837Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:05.184635186Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f09c8d14-475d-49d8-b03f-d3a0b7b880cc, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b202e971-de3b-4a73-a534-ee56c23e79a3, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:05.203235911Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:05.203275793Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b202e971-de3b-4a73-a534-ee56c23e79a3, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b78106a-b661-444a-a40f-011ae4f0911a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:05.223424962Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:05.223466878Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2b78106a-b661-444a-a40f-011ae4f0911a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f64b0db4-4e9f-4666-b267-c9abe9f8b189, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:05.241759004Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:05.241799017Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f64b0db4-4e9f-4666-b267-c9abe9f8b189, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56d79ac6-bb86-49e7-905e-343e52dbbb90, dataset_name: DatasetName { pool_name: ZpoolName { id: bd06e234-5970-4365-a0e5-6cd29811daea (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:05.260836476Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:05.260876602Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 56d79ac6-bb86-49e7-905e-343e52dbbb90, dataset_name: DatasetName { pool_name: ZpoolName { id: bd06e234-5970-4365-a0e5-6cd29811daea (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 010164b7-56ba-4dda-8f67-3d5e05751e2e, dataset_name: DatasetName { pool_name: ZpoolName { id: e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:05.280063945Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:05.280108946Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 010164b7-56ba-4dda-8f67-3d5e05751e2e, dataset_name: DatasetName { pool_name: ZpoolName { id: e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3488ba32-8748-4793-9389-5ea882fa963c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:05.299072111Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:05.299111299Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3488ba32-8748-4793-9389-5ea882fa963c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89e132b1-d133-478e-a8f7-cf9f30b211c9, dataset_name: DatasetName { pool_name: ZpoolName { id: cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:05.316933863Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:05.316982577Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 89e132b1-d133-478e-a8f7-cf9f30b211c9, dataset_name: DatasetName { pool_name: ZpoolName { id: cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:05.333896794Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/968bbfe8-8122-4a85-96dd-954188a2f89d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:05.333944617Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/e8396e20-0493-4030-b55b-5c9e59a237cf/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:05.334151387Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:05.334289625Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:05.427058976Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:05.427180986Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:05.427186152Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_f09c8d14-475d-49d8-b03f-d3a0b7b880cc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:05.427191046Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_b202e971-de3b-4a73-a534-ee56c23e79a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:05.427195199Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:05.42719998Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:05.427204161Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:05.427208099Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:05.42721183Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:05.427215778Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:05.427220344Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:05.427224169Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:05.427227244Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:05.4272312Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:05.427235325Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:05.427240059Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:05.427243922Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:05.427259326Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:05.427263516Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:05.427354099Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"5778ce25-4c87-462f-b652-b2b5c2142a02","remote_addr":"[fd00:1122:3344:101::1]:39162","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94\", TimeNotSynchronized), (\"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93\", TimeNotSynchronized), (\"oxz_crucible_f09c8d14-475d-49d8-b03f-d3a0b7b880cc\", TimeNotSynchronized), (\"oxz_crucible_b202e971-de3b-4a73-a534-ee56c23e79a3\", TimeNotSynchronized), (\"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3\", TimeNotSynchronized), (\"oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139\", TimeNotSynchronized), (\"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f\", TimeNotSynchronized), (\"oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a\", TimeNotSynchronized), (\"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752\", TimeNotSynchronized), (\"oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90\", TimeNotSynchronized), (\"oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e\", TimeNotSynchronized), (\"oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c\", TimeNotSynchronized), (\"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075\", TimeNotSynchronized), (\"oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4\", TimeNotSynchronized), (\"oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9\", TimeNotSynchronized), (\"oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189\", TimeNotSynchronized), (\"oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02\", TimeNotSynchronized), (\"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717\", TimeNotSynchronized), (\"oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02\", TimeNotSynchronized)]","latency_us":471991,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:05.427538642Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"510.038526ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"5778ce25-4c87-462f-b652-b2b5c2142a02\", \"content-length\": \"124\", \"date\": \"Wed, 26 Jun 2024 05:36:04 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"5778ce25-4c87-462f-b652-b2b5c2142a02\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 9f65db5e-d949-4342-9d90-6d691adea120, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: a4d358ae-9d04-40a4-b4a5-ac8418ace88d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 707beb4d-acae-4ff1-bb7e-97992526c5fb, kind: Service { id: a4d358ae-9d04-40a4-b4a5-ac8418ace88d }, name: Name(\"ntp-a4d358ae-9d04-40a4-b4a5-ac8418ace88d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 238, 27])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e3552cf6-27c7-4100-942b-c5877c673046, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } } } }, OmicronZoneConfig { id: 9a0125d1-b636-4039-9e54-dea22726a38a, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } } } }, OmicronZoneConfig { id: 7c851202-bc92-4a32-b946-68379ad242d7, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 653e8bac-3f5b-49fe-a9f9-e9f0546059e9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External } } } }, OmicronZoneConfig { id: e4ffe9aa-395e-4536-a4d0-5cf0bc814584, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 48ce280c-ebea-49f3-9829-dc9b4d018a02, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: c2791662-4c87-4a63-a67a-4d6489fb4eaf, kind: Service { id: 48ce280c-ebea-49f3-9829-dc9b4d018a02 }, name: Name(\"external-dns-48ce280c-ebea-49f3-9829-dc9b4d018a02\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 86])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 38cec1c6-d097-4604-b9cb-30a5fab57e35, kind: Service { id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94 }, name: Name(\"external-dns-9f29eb53-b3d5-4fc1-b4d8-a252757e0c94\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 215])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 67179f00-d639-4f5b-b3c6-71d676113139, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: a724cf8c-c466-4981-aa80-a17ab2349f83, kind: Service { id: 67179f00-d639-4f5b-b3c6-71d676113139 }, name: Name(\"nexus-67179f00-d639-4f5b-b3c6-71d676113139\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 55])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7e2a6156-04b4-4797-8bed-d0c5c9cbe752, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: e31530f0-72e9-4544-bfbb-847c405d46c8, kind: Service { id: 7e2a6156-04b4-4797-8bed-d0c5c9cbe752 }, name: Name(\"nexus-7e2a6156-04b4-4797-8bed-d0c5c9cbe752\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 20a0479c-19b7-40a0-a935-5ff5399f7d02, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 79c95d4d-cff5-4419-8f3e-a6e216736b73, kind: Service { id: 20a0479c-19b7-40a0-a935-5ff5399f7d02 }, name: Name(\"nexus-20a0479c-19b7-40a0-a935-5ff5399f7d02\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 249])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: e42e8525-00d2-488f-a732-5c9514a5c7b4, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 96bb9301-f50f-4259-8a1b-169ec98cbff3, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } } } }, OmicronZoneConfig { id: 966bee63-3642-4e7a-a159-17eb59b72717, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: da8be60e-c115-4289-8526-a1ad7b366b93, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: d72762b4-6a43-4e1c-92b1-b432c1731075, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: c3d43ade-9569-403a-a6cc-494ec947793f, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } } } }, OmicronZoneConfig { id: f09c8d14-475d-49d8-b03f-d3a0b7b880cc, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } } } }, OmicronZoneConfig { id: b202e971-de3b-4a73-a534-ee56c23e79a3, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2b78106a-b661-444a-a40f-011ae4f0911a, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External } } } }, OmicronZoneConfig { id: f64b0db4-4e9f-4666-b267-c9abe9f8b189, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 56d79ac6-bb86-49e7-905e-343e52dbbb90, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd06e234-5970-4365-a0e5-6cd29811daea (zpool), kind: External } } } }, OmicronZoneConfig { id: 010164b7-56ba-4dda-8f67-3d5e05751e2e, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3488ba32-8748-4793-9389-5ea882fa963c, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool), kind: External } } } }, OmicronZoneConfig { id: 89e132b1-d133-478e-a8f7-cf9f30b211c9, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:05.940814855Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:05.941769095Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:05.941776042Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9f65db5e-d949-4342-9d90-6d691adea120, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:05.963352307Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:05.963391598Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9f65db5e-d949-4342-9d90-6d691adea120, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:05.984572614Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:05.984603825Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e3552cf6-27c7-4100-942b-c5877c673046, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:06.003040737Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:06.003143401Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e3552cf6-27c7-4100-942b-c5877c673046, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a0125d1-b636-4039-9e54-dea22726a38a, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:06.022635888Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:06.022668702Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a0125d1-b636-4039-9e54-dea22726a38a, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c851202-bc92-4a32-b946-68379ad242d7, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:06.041048134Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:06.041081125Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7c851202-bc92-4a32-b946-68379ad242d7, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 653e8bac-3f5b-49fe-a9f9-e9f0546059e9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:06.060554072Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:06.060591067Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 653e8bac-3f5b-49fe-a9f9-e9f0546059e9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e4ffe9aa-395e-4536-a4d0-5cf0bc814584, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:06.078878581Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:06.078921575Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e4ffe9aa-395e-4536-a4d0-5cf0bc814584, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48ce280c-ebea-49f3-9829-dc9b4d018a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:06.098637249Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:06.098674947Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 48ce280c-ebea-49f3-9829-dc9b4d018a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:06.11790305Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:06.117940494Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 96bb9301-f50f-4259-8a1b-169ec98cbff3, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:06.137652053Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:06.137684004Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 96bb9301-f50f-4259-8a1b-169ec98cbff3, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c3d43ade-9569-403a-a6cc-494ec947793f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:06.156554317Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:06.156653609Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c3d43ade-9569-403a-a6cc-494ec947793f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f09c8d14-475d-49d8-b03f-d3a0b7b880cc, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:06.175199792Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:06.175231376Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f09c8d14-475d-49d8-b03f-d3a0b7b880cc, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b202e971-de3b-4a73-a534-ee56c23e79a3, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:06.194068071Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:06.194101869Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b202e971-de3b-4a73-a534-ee56c23e79a3, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b78106a-b661-444a-a40f-011ae4f0911a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:06.212475167Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:06.212507699Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2b78106a-b661-444a-a40f-011ae4f0911a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f64b0db4-4e9f-4666-b267-c9abe9f8b189, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:06.232366952Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:06.232402109Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f64b0db4-4e9f-4666-b267-c9abe9f8b189, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56d79ac6-bb86-49e7-905e-343e52dbbb90, dataset_name: DatasetName { pool_name: ZpoolName { id: bd06e234-5970-4365-a0e5-6cd29811daea (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:06.250393847Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:06.250428864Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 56d79ac6-bb86-49e7-905e-343e52dbbb90, dataset_name: DatasetName { pool_name: ZpoolName { id: bd06e234-5970-4365-a0e5-6cd29811daea (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 010164b7-56ba-4dda-8f67-3d5e05751e2e, dataset_name: DatasetName { pool_name: ZpoolName { id: e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:06.270701384Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:06.270735838Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 010164b7-56ba-4dda-8f67-3d5e05751e2e, dataset_name: DatasetName { pool_name: ZpoolName { id: e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3488ba32-8748-4793-9389-5ea882fa963c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:06.290749812Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:06.29078946Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3488ba32-8748-4793-9389-5ea882fa963c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89e132b1-d133-478e-a8f7-cf9f30b211c9, dataset_name: DatasetName { pool_name: ZpoolName { id: cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:06.310677854Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:06.310714464Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 89e132b1-d133-478e-a8f7-cf9f30b211c9, dataset_name: DatasetName { pool_name: ZpoolName { id: cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:06.330518942Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/968bbfe8-8122-4a85-96dd-954188a2f89d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:06.330555787Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/e8396e20-0493-4030-b55b-5c9e59a237cf/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:06.330778494Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:06.330898582Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:06.429203135Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:06.42925467Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:06.429260595Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:06.429264533Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:06.429268106Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:06.429271583Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:06.429275193Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:06.429278934Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:06.429282168Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:06.429285712Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:06.429353485Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_f09c8d14-475d-49d8-b03f-d3a0b7b880cc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:06.429361736Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:06.429365701Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:06.429371908Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:06.429375545Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:06.429379164Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_b202e971-de3b-4a73-a534-ee56c23e79a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:06.42938914Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:06.42939408Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:06.42939874Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:06.42959956Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"b9fb5958-51ce-4cf7-a93f-adae5720ddc7","remote_addr":"[fd00:1122:3344:101::1]:39162","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717\", TimeNotSynchronized), (\"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93\", TimeNotSynchronized), (\"oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9\", TimeNotSynchronized), (\"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3\", TimeNotSynchronized), (\"oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139\", TimeNotSynchronized), (\"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752\", TimeNotSynchronized), (\"oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94\", TimeNotSynchronized), (\"oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02\", TimeNotSynchronized), (\"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f\", TimeNotSynchronized), (\"oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4\", TimeNotSynchronized), (\"oxz_crucible_f09c8d14-475d-49d8-b03f-d3a0b7b880cc\", TimeNotSynchronized), (\"oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a\", TimeNotSynchronized), (\"oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90\", TimeNotSynchronized), (\"oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02\", TimeNotSynchronized), (\"oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e\", TimeNotSynchronized), (\"oxz_crucible_b202e971-de3b-4a73-a534-ee56c23e79a3\", TimeNotSynchronized), (\"oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189\", TimeNotSynchronized), (\"oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c\", TimeNotSynchronized), (\"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075\", TimeNotSynchronized)]","latency_us":486920,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:06.430285311Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"972.520764ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b9fb5958-51ce-4cf7-a93f-adae5720ddc7\", \"content-length\": \"124\", \"date\": \"Wed, 26 Jun 2024 05:36:06 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b9fb5958-51ce-4cf7-a93f-adae5720ddc7\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 9f65db5e-d949-4342-9d90-6d691adea120, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: a4d358ae-9d04-40a4-b4a5-ac8418ace88d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 707beb4d-acae-4ff1-bb7e-97992526c5fb, kind: Service { id: a4d358ae-9d04-40a4-b4a5-ac8418ace88d }, name: Name(\"ntp-a4d358ae-9d04-40a4-b4a5-ac8418ace88d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 238, 27])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e3552cf6-27c7-4100-942b-c5877c673046, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } } } }, OmicronZoneConfig { id: 9a0125d1-b636-4039-9e54-dea22726a38a, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } } } }, OmicronZoneConfig { id: 7c851202-bc92-4a32-b946-68379ad242d7, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 653e8bac-3f5b-49fe-a9f9-e9f0546059e9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External } } } }, OmicronZoneConfig { id: e4ffe9aa-395e-4536-a4d0-5cf0bc814584, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 48ce280c-ebea-49f3-9829-dc9b4d018a02, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: c2791662-4c87-4a63-a67a-4d6489fb4eaf, kind: Service { id: 48ce280c-ebea-49f3-9829-dc9b4d018a02 }, name: Name(\"external-dns-48ce280c-ebea-49f3-9829-dc9b4d018a02\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 86])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 38cec1c6-d097-4604-b9cb-30a5fab57e35, kind: Service { id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94 }, name: Name(\"external-dns-9f29eb53-b3d5-4fc1-b4d8-a252757e0c94\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 215])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 67179f00-d639-4f5b-b3c6-71d676113139, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: a724cf8c-c466-4981-aa80-a17ab2349f83, kind: Service { id: 67179f00-d639-4f5b-b3c6-71d676113139 }, name: Name(\"nexus-67179f00-d639-4f5b-b3c6-71d676113139\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 55])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7e2a6156-04b4-4797-8bed-d0c5c9cbe752, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: e31530f0-72e9-4544-bfbb-847c405d46c8, kind: Service { id: 7e2a6156-04b4-4797-8bed-d0c5c9cbe752 }, name: Name(\"nexus-7e2a6156-04b4-4797-8bed-d0c5c9cbe752\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 20a0479c-19b7-40a0-a935-5ff5399f7d02, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 79c95d4d-cff5-4419-8f3e-a6e216736b73, kind: Service { id: 20a0479c-19b7-40a0-a935-5ff5399f7d02 }, name: Name(\"nexus-20a0479c-19b7-40a0-a935-5ff5399f7d02\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 249])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: e42e8525-00d2-488f-a732-5c9514a5c7b4, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 96bb9301-f50f-4259-8a1b-169ec98cbff3, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } } } }, OmicronZoneConfig { id: 966bee63-3642-4e7a-a159-17eb59b72717, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: da8be60e-c115-4289-8526-a1ad7b366b93, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: d72762b4-6a43-4e1c-92b1-b432c1731075, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: c3d43ade-9569-403a-a6cc-494ec947793f, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } } } }, OmicronZoneConfig { id: f09c8d14-475d-49d8-b03f-d3a0b7b880cc, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } } } }, OmicronZoneConfig { id: b202e971-de3b-4a73-a534-ee56c23e79a3, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2b78106a-b661-444a-a40f-011ae4f0911a, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External } } } }, OmicronZoneConfig { id: f64b0db4-4e9f-4666-b267-c9abe9f8b189, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 56d79ac6-bb86-49e7-905e-343e52dbbb90, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd06e234-5970-4365-a0e5-6cd29811daea (zpool), kind: External } } } }, OmicronZoneConfig { id: 010164b7-56ba-4dda-8f67-3d5e05751e2e, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3488ba32-8748-4793-9389-5ea882fa963c, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool), kind: External } } } }, OmicronZoneConfig { id: 89e132b1-d133-478e-a8f7-cf9f30b211c9, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:07.407721047Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:07.408096739Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:07.408105861Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9f65db5e-d949-4342-9d90-6d691adea120, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:07.430194465Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:07.430240254Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9f65db5e-d949-4342-9d90-6d691adea120, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:07.451631959Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:07.451674757Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e3552cf6-27c7-4100-942b-c5877c673046, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:07.47126496Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:07.471315155Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e3552cf6-27c7-4100-942b-c5877c673046, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a0125d1-b636-4039-9e54-dea22726a38a, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:07.491013934Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:07.491059732Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a0125d1-b636-4039-9e54-dea22726a38a, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c851202-bc92-4a32-b946-68379ad242d7, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:07.511618556Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:07.511657876Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7c851202-bc92-4a32-b946-68379ad242d7, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 653e8bac-3f5b-49fe-a9f9-e9f0546059e9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:07.530844316Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:07.530882811Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 653e8bac-3f5b-49fe-a9f9-e9f0546059e9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e4ffe9aa-395e-4536-a4d0-5cf0bc814584, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:07.550065261Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:07.550134488Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e4ffe9aa-395e-4536-a4d0-5cf0bc814584, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48ce280c-ebea-49f3-9829-dc9b4d018a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:07.572469392Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:07.572581174Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 48ce280c-ebea-49f3-9829-dc9b4d018a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:07.593533471Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:07.593646585Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 96bb9301-f50f-4259-8a1b-169ec98cbff3, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:07.615850611Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:07.615892444Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 96bb9301-f50f-4259-8a1b-169ec98cbff3, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c3d43ade-9569-403a-a6cc-494ec947793f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:07.636693666Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:07.636798516Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c3d43ade-9569-403a-a6cc-494ec947793f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f09c8d14-475d-49d8-b03f-d3a0b7b880cc, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:07.658412198Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:07.658459787Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f09c8d14-475d-49d8-b03f-d3a0b7b880cc, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b202e971-de3b-4a73-a534-ee56c23e79a3, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:07.679454997Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:07.679493661Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b202e971-de3b-4a73-a534-ee56c23e79a3, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b78106a-b661-444a-a40f-011ae4f0911a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:07.700233909Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:07.700276379Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2b78106a-b661-444a-a40f-011ae4f0911a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f64b0db4-4e9f-4666-b267-c9abe9f8b189, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:07.719496616Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:07.719534004Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f64b0db4-4e9f-4666-b267-c9abe9f8b189, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56d79ac6-bb86-49e7-905e-343e52dbbb90, dataset_name: DatasetName { pool_name: ZpoolName { id: bd06e234-5970-4365-a0e5-6cd29811daea (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:07.738854368Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:07.738891869Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 56d79ac6-bb86-49e7-905e-343e52dbbb90, dataset_name: DatasetName { pool_name: ZpoolName { id: bd06e234-5970-4365-a0e5-6cd29811daea (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 010164b7-56ba-4dda-8f67-3d5e05751e2e, dataset_name: DatasetName { pool_name: ZpoolName { id: e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:07.760056054Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:07.76009692Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 010164b7-56ba-4dda-8f67-3d5e05751e2e, dataset_name: DatasetName { pool_name: ZpoolName { id: e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3488ba32-8748-4793-9389-5ea882fa963c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:07.780740662Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:07.780781651Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3488ba32-8748-4793-9389-5ea882fa963c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89e132b1-d133-478e-a8f7-cf9f30b211c9, dataset_name: DatasetName { pool_name: ZpoolName { id: cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:07.801440676Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:07.801490328Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 89e132b1-d133-478e-a8f7-cf9f30b211c9, dataset_name: DatasetName { pool_name: ZpoolName { id: cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:07.822079349Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/968bbfe8-8122-4a85-96dd-954188a2f89d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:07.822131063Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/e8396e20-0493-4030-b55b-5c9e59a237cf/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:07.822319983Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:07.822519244Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:07.919816462Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:07.919868926Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:07.919874204Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:07.919878413Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:07.91988236Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:07.919886701Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:07.919890854Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:07.919894566Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:07.919897604Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_b202e971-de3b-4a73-a534-ee56c23e79a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:07.919900492Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:07.919903408Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_f09c8d14-475d-49d8-b03f-d3a0b7b880cc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:07.919906286Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:07.919909211Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:07.919912164Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:07.919915051Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:07.91991793Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:07.919927914Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:07.91993113Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:07.920080918Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:07.920206086Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"676050a9-9b63-4472-be1c-db30df6d3795","remote_addr":"[fd00:1122:3344:101::1]:39162","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02\", TimeNotSynchronized), (\"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717\", TimeNotSynchronized), (\"oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a\", TimeNotSynchronized), (\"oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9\", TimeNotSynchronized), (\"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3\", TimeNotSynchronized), (\"oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02\", TimeNotSynchronized), (\"oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c\", TimeNotSynchronized), (\"oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139\", TimeNotSynchronized), (\"oxz_crucible_b202e971-de3b-4a73-a534-ee56c23e79a3\", TimeNotSynchronized), (\"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752\", TimeNotSynchronized), (\"oxz_crucible_f09c8d14-475d-49d8-b03f-d3a0b7b880cc\", TimeNotSynchronized), (\"oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189\", TimeNotSynchronized), (\"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075\", TimeNotSynchronized), (\"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f\", TimeNotSynchronized), (\"oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4\", TimeNotSynchronized), (\"oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90\", TimeNotSynchronized), (\"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93\", TimeNotSynchronized), (\"oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e\", TimeNotSynchronized), (\"oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94\", TimeNotSynchronized)]","latency_us":510554,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:07.920394153Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"1.079717239s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"676050a9-9b63-4472-be1c-db30df6d3795\", \"content-length\": \"124\", \"date\": \"Wed, 26 Jun 2024 05:36:07 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"676050a9-9b63-4472-be1c-db30df6d3795\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 9f65db5e-d949-4342-9d90-6d691adea120, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: a4d358ae-9d04-40a4-b4a5-ac8418ace88d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 707beb4d-acae-4ff1-bb7e-97992526c5fb, kind: Service { id: a4d358ae-9d04-40a4-b4a5-ac8418ace88d }, name: Name(\"ntp-a4d358ae-9d04-40a4-b4a5-ac8418ace88d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 238, 27])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e3552cf6-27c7-4100-942b-c5877c673046, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } } } }, OmicronZoneConfig { id: 9a0125d1-b636-4039-9e54-dea22726a38a, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } } } }, OmicronZoneConfig { id: 7c851202-bc92-4a32-b946-68379ad242d7, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 653e8bac-3f5b-49fe-a9f9-e9f0546059e9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External } } } }, OmicronZoneConfig { id: e4ffe9aa-395e-4536-a4d0-5cf0bc814584, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 48ce280c-ebea-49f3-9829-dc9b4d018a02, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: c2791662-4c87-4a63-a67a-4d6489fb4eaf, kind: Service { id: 48ce280c-ebea-49f3-9829-dc9b4d018a02 }, name: Name(\"external-dns-48ce280c-ebea-49f3-9829-dc9b4d018a02\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 86])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 38cec1c6-d097-4604-b9cb-30a5fab57e35, kind: Service { id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94 }, name: Name(\"external-dns-9f29eb53-b3d5-4fc1-b4d8-a252757e0c94\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 215])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 67179f00-d639-4f5b-b3c6-71d676113139, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: a724cf8c-c466-4981-aa80-a17ab2349f83, kind: Service { id: 67179f00-d639-4f5b-b3c6-71d676113139 }, name: Name(\"nexus-67179f00-d639-4f5b-b3c6-71d676113139\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 55])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7e2a6156-04b4-4797-8bed-d0c5c9cbe752, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: e31530f0-72e9-4544-bfbb-847c405d46c8, kind: Service { id: 7e2a6156-04b4-4797-8bed-d0c5c9cbe752 }, name: Name(\"nexus-7e2a6156-04b4-4797-8bed-d0c5c9cbe752\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 20a0479c-19b7-40a0-a935-5ff5399f7d02, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 79c95d4d-cff5-4419-8f3e-a6e216736b73, kind: Service { id: 20a0479c-19b7-40a0-a935-5ff5399f7d02 }, name: Name(\"nexus-20a0479c-19b7-40a0-a935-5ff5399f7d02\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 249])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: e42e8525-00d2-488f-a732-5c9514a5c7b4, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 96bb9301-f50f-4259-8a1b-169ec98cbff3, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } } } }, OmicronZoneConfig { id: 966bee63-3642-4e7a-a159-17eb59b72717, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: da8be60e-c115-4289-8526-a1ad7b366b93, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: d72762b4-6a43-4e1c-92b1-b432c1731075, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: c3d43ade-9569-403a-a6cc-494ec947793f, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } } } }, OmicronZoneConfig { id: f09c8d14-475d-49d8-b03f-d3a0b7b880cc, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } } } }, OmicronZoneConfig { id: b202e971-de3b-4a73-a534-ee56c23e79a3, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2b78106a-b661-444a-a40f-011ae4f0911a, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External } } } }, OmicronZoneConfig { id: f64b0db4-4e9f-4666-b267-c9abe9f8b189, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 56d79ac6-bb86-49e7-905e-343e52dbbb90, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd06e234-5970-4365-a0e5-6cd29811daea (zpool), kind: External } } } }, OmicronZoneConfig { id: 010164b7-56ba-4dda-8f67-3d5e05751e2e, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3488ba32-8748-4793-9389-5ea882fa963c, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool), kind: External } } } }, OmicronZoneConfig { id: 89e132b1-d133-478e-a8f7-cf9f30b211c9, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:09.005026477Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:09.00539347Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:09.005403726Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9f65db5e-d949-4342-9d90-6d691adea120, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:09.027052049Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:09.027092493Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9f65db5e-d949-4342-9d90-6d691adea120, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:09.046290029Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:09.046336249Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e3552cf6-27c7-4100-942b-c5877c673046, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:09.066412387Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:09.066447947Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e3552cf6-27c7-4100-942b-c5877c673046, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a0125d1-b636-4039-9e54-dea22726a38a, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:09.084969854Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:09.085001214Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a0125d1-b636-4039-9e54-dea22726a38a, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c851202-bc92-4a32-b946-68379ad242d7, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:09.103145532Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:09.103183567Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7c851202-bc92-4a32-b946-68379ad242d7, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 653e8bac-3f5b-49fe-a9f9-e9f0546059e9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:09.121669441Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:09.121706219Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 653e8bac-3f5b-49fe-a9f9-e9f0546059e9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e4ffe9aa-395e-4536-a4d0-5cf0bc814584, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:09.142502497Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:09.142540514Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e4ffe9aa-395e-4536-a4d0-5cf0bc814584, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48ce280c-ebea-49f3-9829-dc9b4d018a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:09.161061986Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:09.16109494Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 48ce280c-ebea-49f3-9829-dc9b4d018a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:09.183951198Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:09.183987002Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 96bb9301-f50f-4259-8a1b-169ec98cbff3, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:09.20455171Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:09.204583651Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 96bb9301-f50f-4259-8a1b-169ec98cbff3, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c3d43ade-9569-403a-a6cc-494ec947793f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:09.225405279Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:09.225441082Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c3d43ade-9569-403a-a6cc-494ec947793f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f09c8d14-475d-49d8-b03f-d3a0b7b880cc, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:09.245259717Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:09.245357797Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f09c8d14-475d-49d8-b03f-d3a0b7b880cc, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b202e971-de3b-4a73-a534-ee56c23e79a3, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:09.265439238Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:09.26547042Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b202e971-de3b-4a73-a534-ee56c23e79a3, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b78106a-b661-444a-a40f-011ae4f0911a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:09.287347136Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:09.287379518Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2b78106a-b661-444a-a40f-011ae4f0911a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f64b0db4-4e9f-4666-b267-c9abe9f8b189, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:09.307663467Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:09.307695549Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f64b0db4-4e9f-4666-b267-c9abe9f8b189, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56d79ac6-bb86-49e7-905e-343e52dbbb90, dataset_name: DatasetName { pool_name: ZpoolName { id: bd06e234-5970-4365-a0e5-6cd29811daea (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:09.329234257Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:09.329276353Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 56d79ac6-bb86-49e7-905e-343e52dbbb90, dataset_name: DatasetName { pool_name: ZpoolName { id: bd06e234-5970-4365-a0e5-6cd29811daea (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 010164b7-56ba-4dda-8f67-3d5e05751e2e, dataset_name: DatasetName { pool_name: ZpoolName { id: e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:09.352905707Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:09.352941436Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 010164b7-56ba-4dda-8f67-3d5e05751e2e, dataset_name: DatasetName { pool_name: ZpoolName { id: e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3488ba32-8748-4793-9389-5ea882fa963c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:09.378370378Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:09.37842603Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3488ba32-8748-4793-9389-5ea882fa963c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89e132b1-d133-478e-a8f7-cf9f30b211c9, dataset_name: DatasetName { pool_name: ZpoolName { id: cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:09.400431568Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:09.40047414Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 89e132b1-d133-478e-a8f7-cf9f30b211c9, dataset_name: DatasetName { pool_name: ZpoolName { id: cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:09.42431778Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/968bbfe8-8122-4a85-96dd-954188a2f89d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:09.424358281Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/e8396e20-0493-4030-b55b-5c9e59a237cf/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:09.424550379Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:09.424662807Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:09.525015321Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:09.52508631Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:09.525094073Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:09.525100711Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:09.525105389Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:09.525110292Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:09.525116377Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:09.52512113Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:09.525126164Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_f09c8d14-475d-49d8-b03f-d3a0b7b880cc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:09.525131105Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:09.525137884Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:09.525142806Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:09.525148824Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:09.525153868Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_b202e971-de3b-4a73-a534-ee56c23e79a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:09.525160112Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:09.525164912Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:09.525173622Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:09.525178666Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:09.525182078Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:09.525321955Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"36d4d086-a56c-4ff0-92c4-3da74065da07","remote_addr":"[fd00:1122:3344:101::1]:39162","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02\", TimeNotSynchronized), (\"oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4\", TimeNotSynchronized), (\"oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a\", TimeNotSynchronized), (\"oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189\", TimeNotSynchronized), (\"oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94\", TimeNotSynchronized), (\"oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90\", TimeNotSynchronized), (\"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f\", TimeNotSynchronized), (\"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075\", TimeNotSynchronized), (\"oxz_crucible_f09c8d14-475d-49d8-b03f-d3a0b7b880cc\", TimeNotSynchronized), (\"oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139\", TimeNotSynchronized), (\"oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02\", TimeNotSynchronized), (\"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717\", TimeNotSynchronized), (\"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3\", TimeNotSynchronized), (\"oxz_crucible_b202e971-de3b-4a73-a534-ee56c23e79a3\", TimeNotSynchronized), (\"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752\", TimeNotSynchronized), (\"oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c\", TimeNotSynchronized), (\"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93\", TimeNotSynchronized), (\"oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9\", TimeNotSynchronized), (\"oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e\", TimeNotSynchronized)]","latency_us":518278,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:09.525511315Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"634.98394ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"36d4d086-a56c-4ff0-92c4-3da74065da07\", \"content-length\": \"124\", \"date\": \"Wed, 26 Jun 2024 05:36:09 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"36d4d086-a56c-4ff0-92c4-3da74065da07\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 9f65db5e-d949-4342-9d90-6d691adea120, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: a4d358ae-9d04-40a4-b4a5-ac8418ace88d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 707beb4d-acae-4ff1-bb7e-97992526c5fb, kind: Service { id: a4d358ae-9d04-40a4-b4a5-ac8418ace88d }, name: Name(\"ntp-a4d358ae-9d04-40a4-b4a5-ac8418ace88d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 238, 27])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e3552cf6-27c7-4100-942b-c5877c673046, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } } } }, OmicronZoneConfig { id: 9a0125d1-b636-4039-9e54-dea22726a38a, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } } } }, OmicronZoneConfig { id: 7c851202-bc92-4a32-b946-68379ad242d7, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 653e8bac-3f5b-49fe-a9f9-e9f0546059e9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External } } } }, OmicronZoneConfig { id: e4ffe9aa-395e-4536-a4d0-5cf0bc814584, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 48ce280c-ebea-49f3-9829-dc9b4d018a02, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: c2791662-4c87-4a63-a67a-4d6489fb4eaf, kind: Service { id: 48ce280c-ebea-49f3-9829-dc9b4d018a02 }, name: Name(\"external-dns-48ce280c-ebea-49f3-9829-dc9b4d018a02\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 86])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 38cec1c6-d097-4604-b9cb-30a5fab57e35, kind: Service { id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94 }, name: Name(\"external-dns-9f29eb53-b3d5-4fc1-b4d8-a252757e0c94\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 215])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 67179f00-d639-4f5b-b3c6-71d676113139, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: a724cf8c-c466-4981-aa80-a17ab2349f83, kind: Service { id: 67179f00-d639-4f5b-b3c6-71d676113139 }, name: Name(\"nexus-67179f00-d639-4f5b-b3c6-71d676113139\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 55])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7e2a6156-04b4-4797-8bed-d0c5c9cbe752, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: e31530f0-72e9-4544-bfbb-847c405d46c8, kind: Service { id: 7e2a6156-04b4-4797-8bed-d0c5c9cbe752 }, name: Name(\"nexus-7e2a6156-04b4-4797-8bed-d0c5c9cbe752\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 20a0479c-19b7-40a0-a935-5ff5399f7d02, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 79c95d4d-cff5-4419-8f3e-a6e216736b73, kind: Service { id: 20a0479c-19b7-40a0-a935-5ff5399f7d02 }, name: Name(\"nexus-20a0479c-19b7-40a0-a935-5ff5399f7d02\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 249])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: e42e8525-00d2-488f-a732-5c9514a5c7b4, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 96bb9301-f50f-4259-8a1b-169ec98cbff3, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } } } }, OmicronZoneConfig { id: 966bee63-3642-4e7a-a159-17eb59b72717, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: da8be60e-c115-4289-8526-a1ad7b366b93, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: d72762b4-6a43-4e1c-92b1-b432c1731075, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: c3d43ade-9569-403a-a6cc-494ec947793f, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } } } }, OmicronZoneConfig { id: f09c8d14-475d-49d8-b03f-d3a0b7b880cc, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } } } }, OmicronZoneConfig { id: b202e971-de3b-4a73-a534-ee56c23e79a3, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2b78106a-b661-444a-a40f-011ae4f0911a, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External } } } }, OmicronZoneConfig { id: f64b0db4-4e9f-4666-b267-c9abe9f8b189, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 56d79ac6-bb86-49e7-905e-343e52dbbb90, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd06e234-5970-4365-a0e5-6cd29811daea (zpool), kind: External } } } }, OmicronZoneConfig { id: 010164b7-56ba-4dda-8f67-3d5e05751e2e, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3488ba32-8748-4793-9389-5ea882fa963c, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool), kind: External } } } }, OmicronZoneConfig { id: 89e132b1-d133-478e-a8f7-cf9f30b211c9, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:10.164209331Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:10.164613361Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:10.164620327Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9f65db5e-d949-4342-9d90-6d691adea120, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:10.185158717Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:10.185210047Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9f65db5e-d949-4342-9d90-6d691adea120, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:10.192794564Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:423","component":"producer-server","component":"MetricsManager","sled_id":"8ead6dd9-9765-4776-9768-713d8773d37f","component":"SledAgent","file":"oximeter/producer/src/lib.rs:423","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"2.579380885s"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:10.205972988Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:10.206016189Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e3552cf6-27c7-4100-942b-c5877c673046, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:10.226446067Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:10.22649569Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e3552cf6-27c7-4100-942b-c5877c673046, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a0125d1-b636-4039-9e54-dea22726a38a, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:10.245447512Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:10.245486523Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a0125d1-b636-4039-9e54-dea22726a38a, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c851202-bc92-4a32-b946-68379ad242d7, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:10.263795778Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:10.263908715Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7c851202-bc92-4a32-b946-68379ad242d7, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 653e8bac-3f5b-49fe-a9f9-e9f0546059e9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:10.281956713Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:10.282009551Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 653e8bac-3f5b-49fe-a9f9-e9f0546059e9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e4ffe9aa-395e-4536-a4d0-5cf0bc814584, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:10.300305789Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:10.300350097Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e4ffe9aa-395e-4536-a4d0-5cf0bc814584, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48ce280c-ebea-49f3-9829-dc9b4d018a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:10.319710286Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:10.319755605Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 48ce280c-ebea-49f3-9829-dc9b4d018a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:10.339154354Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:10.339204408Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 96bb9301-f50f-4259-8a1b-169ec98cbff3, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:10.358533741Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:10.358573595Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 96bb9301-f50f-4259-8a1b-169ec98cbff3, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c3d43ade-9569-403a-a6cc-494ec947793f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:10.379097194Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:10.379142139Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c3d43ade-9569-403a-a6cc-494ec947793f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f09c8d14-475d-49d8-b03f-d3a0b7b880cc, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:10.399238433Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:10.399342808Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f09c8d14-475d-49d8-b03f-d3a0b7b880cc, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b202e971-de3b-4a73-a534-ee56c23e79a3, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:10.418159848Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:10.418197742Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b202e971-de3b-4a73-a534-ee56c23e79a3, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b78106a-b661-444a-a40f-011ae4f0911a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:10.437627802Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:10.437666025Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2b78106a-b661-444a-a40f-011ae4f0911a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f64b0db4-4e9f-4666-b267-c9abe9f8b189, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:10.457396075Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:10.457435713Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f64b0db4-4e9f-4666-b267-c9abe9f8b189, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56d79ac6-bb86-49e7-905e-343e52dbbb90, dataset_name: DatasetName { pool_name: ZpoolName { id: bd06e234-5970-4365-a0e5-6cd29811daea (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:10.478643368Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:10.478687357Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 56d79ac6-bb86-49e7-905e-343e52dbbb90, dataset_name: DatasetName { pool_name: ZpoolName { id: bd06e234-5970-4365-a0e5-6cd29811daea (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 010164b7-56ba-4dda-8f67-3d5e05751e2e, dataset_name: DatasetName { pool_name: ZpoolName { id: e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:10.49799719Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:10.498038535Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 010164b7-56ba-4dda-8f67-3d5e05751e2e, dataset_name: DatasetName { pool_name: ZpoolName { id: e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3488ba32-8748-4793-9389-5ea882fa963c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:10.518119033Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:10.518163527Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3488ba32-8748-4793-9389-5ea882fa963c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89e132b1-d133-478e-a8f7-cf9f30b211c9, dataset_name: DatasetName { pool_name: ZpoolName { id: cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:10.537238346Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:10.537285364Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 89e132b1-d133-478e-a8f7-cf9f30b211c9, dataset_name: DatasetName { pool_name: ZpoolName { id: cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:10.556911897Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/968bbfe8-8122-4a85-96dd-954188a2f89d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:10.556963161Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/e8396e20-0493-4030-b55b-5c9e59a237cf/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:10.55718495Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:10.557315012Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:10.654799624Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:10.654841719Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:10.654847503Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:10.654851798Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:10.654856606Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:10.654860695Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:10.654865439Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:10.654869517Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:10.654873398Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:10.65487651Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_f09c8d14-475d-49d8-b03f-d3a0b7b880cc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:10.654881395Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_b202e971-de3b-4a73-a534-ee56c23e79a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:10.654885482Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:10.654889429Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:10.654908415Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:10.654913468Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:10.65491733Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:10.65492139Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:10.654925955Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:10.654929912Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:10.655027752Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"22b835db-99ce-4cb7-ae3e-18c8cf5d41f7","remote_addr":"[fd00:1122:3344:101::1]:39162","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139\", TimeNotSynchronized), (\"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3\", TimeNotSynchronized), (\"oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02\", TimeNotSynchronized), (\"oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02\", TimeNotSynchronized), (\"oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189\", TimeNotSynchronized), (\"oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90\", TimeNotSynchronized), (\"oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c\", TimeNotSynchronized), (\"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752\", TimeNotSynchronized), (\"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f\", TimeNotSynchronized), (\"oxz_crucible_f09c8d14-475d-49d8-b03f-d3a0b7b880cc\", TimeNotSynchronized), (\"oxz_crucible_b202e971-de3b-4a73-a534-ee56c23e79a3\", TimeNotSynchronized), (\"oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9\", TimeNotSynchronized), (\"oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4\", TimeNotSynchronized), (\"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717\", TimeNotSynchronized), (\"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075\", TimeNotSynchronized), (\"oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94\", TimeNotSynchronized), (\"oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a\", TimeNotSynchronized), (\"oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e\", TimeNotSynchronized), (\"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93\", TimeNotSynchronized)]","latency_us":488982,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:10.655292373Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"1.739651101s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"22b835db-99ce-4cb7-ae3e-18c8cf5d41f7\", \"content-length\": \"124\", \"date\": \"Wed, 26 Jun 2024 05:36:10 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"22b835db-99ce-4cb7-ae3e-18c8cf5d41f7\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 9f65db5e-d949-4342-9d90-6d691adea120, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: a4d358ae-9d04-40a4-b4a5-ac8418ace88d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 707beb4d-acae-4ff1-bb7e-97992526c5fb, kind: Service { id: a4d358ae-9d04-40a4-b4a5-ac8418ace88d }, name: Name(\"ntp-a4d358ae-9d04-40a4-b4a5-ac8418ace88d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 238, 27])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e3552cf6-27c7-4100-942b-c5877c673046, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } } } }, OmicronZoneConfig { id: 9a0125d1-b636-4039-9e54-dea22726a38a, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } } } }, OmicronZoneConfig { id: 7c851202-bc92-4a32-b946-68379ad242d7, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 653e8bac-3f5b-49fe-a9f9-e9f0546059e9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External } } } }, OmicronZoneConfig { id: e4ffe9aa-395e-4536-a4d0-5cf0bc814584, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 48ce280c-ebea-49f3-9829-dc9b4d018a02, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: c2791662-4c87-4a63-a67a-4d6489fb4eaf, kind: Service { id: 48ce280c-ebea-49f3-9829-dc9b4d018a02 }, name: Name(\"external-dns-48ce280c-ebea-49f3-9829-dc9b4d018a02\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 86])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 38cec1c6-d097-4604-b9cb-30a5fab57e35, kind: Service { id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94 }, name: Name(\"external-dns-9f29eb53-b3d5-4fc1-b4d8-a252757e0c94\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 215])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 67179f00-d639-4f5b-b3c6-71d676113139, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: a724cf8c-c466-4981-aa80-a17ab2349f83, kind: Service { id: 67179f00-d639-4f5b-b3c6-71d676113139 }, name: Name(\"nexus-67179f00-d639-4f5b-b3c6-71d676113139\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 55])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7e2a6156-04b4-4797-8bed-d0c5c9cbe752, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: e31530f0-72e9-4544-bfbb-847c405d46c8, kind: Service { id: 7e2a6156-04b4-4797-8bed-d0c5c9cbe752 }, name: Name(\"nexus-7e2a6156-04b4-4797-8bed-d0c5c9cbe752\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 20a0479c-19b7-40a0-a935-5ff5399f7d02, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 79c95d4d-cff5-4419-8f3e-a6e216736b73, kind: Service { id: 20a0479c-19b7-40a0-a935-5ff5399f7d02 }, name: Name(\"nexus-20a0479c-19b7-40a0-a935-5ff5399f7d02\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 249])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: e42e8525-00d2-488f-a732-5c9514a5c7b4, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 96bb9301-f50f-4259-8a1b-169ec98cbff3, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } } } }, OmicronZoneConfig { id: 966bee63-3642-4e7a-a159-17eb59b72717, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: da8be60e-c115-4289-8526-a1ad7b366b93, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: d72762b4-6a43-4e1c-92b1-b432c1731075, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: c3d43ade-9569-403a-a6cc-494ec947793f, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } } } }, OmicronZoneConfig { id: f09c8d14-475d-49d8-b03f-d3a0b7b880cc, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } } } }, OmicronZoneConfig { id: b202e971-de3b-4a73-a534-ee56c23e79a3, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2b78106a-b661-444a-a40f-011ae4f0911a, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External } } } }, OmicronZoneConfig { id: f64b0db4-4e9f-4666-b267-c9abe9f8b189, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 56d79ac6-bb86-49e7-905e-343e52dbbb90, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd06e234-5970-4365-a0e5-6cd29811daea (zpool), kind: External } } } }, OmicronZoneConfig { id: 010164b7-56ba-4dda-8f67-3d5e05751e2e, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3488ba32-8748-4793-9389-5ea882fa963c, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool), kind: External } } } }, OmicronZoneConfig { id: 89e132b1-d133-478e-a8f7-cf9f30b211c9, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:12.401702071Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:12.402009915Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:12.402015821Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9f65db5e-d949-4342-9d90-6d691adea120, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:12.424873782Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:12.42491119Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9f65db5e-d949-4342-9d90-6d691adea120, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:12.446559646Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:12.446593753Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e3552cf6-27c7-4100-942b-c5877c673046, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:12.467227184Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:12.467261863Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e3552cf6-27c7-4100-942b-c5877c673046, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a0125d1-b636-4039-9e54-dea22726a38a, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:12.48677095Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:12.486803576Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a0125d1-b636-4039-9e54-dea22726a38a, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c851202-bc92-4a32-b946-68379ad242d7, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:12.506559847Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:12.506661757Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7c851202-bc92-4a32-b946-68379ad242d7, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 653e8bac-3f5b-49fe-a9f9-e9f0546059e9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:12.526316548Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:12.526357864Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 653e8bac-3f5b-49fe-a9f9-e9f0546059e9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e4ffe9aa-395e-4536-a4d0-5cf0bc814584, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:12.546866383Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:12.546899684Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e4ffe9aa-395e-4536-a4d0-5cf0bc814584, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48ce280c-ebea-49f3-9829-dc9b4d018a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:12.565552952Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:12.565581255Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 48ce280c-ebea-49f3-9829-dc9b4d018a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:12.583800129Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:12.583824964Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 96bb9301-f50f-4259-8a1b-169ec98cbff3, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:12.603450336Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:12.603482839Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 96bb9301-f50f-4259-8a1b-169ec98cbff3, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c3d43ade-9569-403a-a6cc-494ec947793f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:12.623741468Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:12.623772846Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c3d43ade-9569-403a-a6cc-494ec947793f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f09c8d14-475d-49d8-b03f-d3a0b7b880cc, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:12.645787588Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:12.64581981Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f09c8d14-475d-49d8-b03f-d3a0b7b880cc, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b202e971-de3b-4a73-a534-ee56c23e79a3, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:12.666612762Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:12.666645322Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b202e971-de3b-4a73-a534-ee56c23e79a3, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b78106a-b661-444a-a40f-011ae4f0911a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:12.687524508Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:12.687691725Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2b78106a-b661-444a-a40f-011ae4f0911a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f64b0db4-4e9f-4666-b267-c9abe9f8b189, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:12.708688703Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:12.708720569Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f64b0db4-4e9f-4666-b267-c9abe9f8b189, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56d79ac6-bb86-49e7-905e-343e52dbbb90, dataset_name: DatasetName { pool_name: ZpoolName { id: bd06e234-5970-4365-a0e5-6cd29811daea (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:12.730183135Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:12.730211505Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 56d79ac6-bb86-49e7-905e-343e52dbbb90, dataset_name: DatasetName { pool_name: ZpoolName { id: bd06e234-5970-4365-a0e5-6cd29811daea (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 010164b7-56ba-4dda-8f67-3d5e05751e2e, dataset_name: DatasetName { pool_name: ZpoolName { id: e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:12.750168303Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:12.750196898Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 010164b7-56ba-4dda-8f67-3d5e05751e2e, dataset_name: DatasetName { pool_name: ZpoolName { id: e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3488ba32-8748-4793-9389-5ea882fa963c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:12.771864303Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:12.77189466Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3488ba32-8748-4793-9389-5ea882fa963c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89e132b1-d133-478e-a8f7-cf9f30b211c9, dataset_name: DatasetName { pool_name: ZpoolName { id: cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:12.793696729Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:12.793745593Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 89e132b1-d133-478e-a8f7-cf9f30b211c9, dataset_name: DatasetName { pool_name: ZpoolName { id: cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:12.817316679Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/968bbfe8-8122-4a85-96dd-954188a2f89d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:12.81735311Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/e8396e20-0493-4030-b55b-5c9e59a237cf/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:12.81757401Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:12.817783345Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:12.917112184Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:12.917165548Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:12.917173489Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:12.917179002Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:12.917185658Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:12.91719102Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:12.917196533Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:12.917201567Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:12.917205842Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:12.917211364Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_f09c8d14-475d-49d8-b03f-d3a0b7b880cc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:12.917216511Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:12.917222006Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:12.917227434Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:12.917243521Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:12.917250196Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_b202e971-de3b-4a73-a534-ee56c23e79a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:12.917256843Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:12.917261972Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:12.917266978Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:12.917272003Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:12.917401709Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"2012aa20-511b-449e-960d-3687f19d08f9","remote_addr":"[fd00:1122:3344:101::1]:39162","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02\", TimeNotSynchronized), (\"oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02\", TimeNotSynchronized), (\"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717\", TimeNotSynchronized), (\"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f\", TimeNotSynchronized), (\"oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a\", TimeNotSynchronized), (\"oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90\", TimeNotSynchronized), (\"oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e\", TimeNotSynchronized), (\"oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94\", TimeNotSynchronized), (\"oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4\", TimeNotSynchronized), (\"oxz_crucible_f09c8d14-475d-49d8-b03f-d3a0b7b880cc\", TimeNotSynchronized), (\"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3\", TimeNotSynchronized), (\"oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139\", TimeNotSynchronized), (\"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075\", TimeNotSynchronized), (\"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93\", TimeNotSynchronized), (\"oxz_crucible_b202e971-de3b-4a73-a534-ee56c23e79a3\", TimeNotSynchronized), (\"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752\", TimeNotSynchronized), (\"oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c\", TimeNotSynchronized), (\"oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189\", TimeNotSynchronized), (\"oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9\", TimeNotSynchronized)]","latency_us":513764,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:12.917604842Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"2.282991844s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"2012aa20-511b-449e-960d-3687f19d08f9\", \"content-length\": \"124\", \"date\": \"Wed, 26 Jun 2024 05:36:12 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"2012aa20-511b-449e-960d-3687f19d08f9\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:14.792666039Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:54916"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:14.793624883Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"c7ecd0a2-aa33-4e38-af8a-37f0af06f4f9","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:54916","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":42,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 9f65db5e-d949-4342-9d90-6d691adea120, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: a4d358ae-9d04-40a4-b4a5-ac8418ace88d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 707beb4d-acae-4ff1-bb7e-97992526c5fb, kind: Service { id: a4d358ae-9d04-40a4-b4a5-ac8418ace88d }, name: Name(\"ntp-a4d358ae-9d04-40a4-b4a5-ac8418ace88d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 238, 27])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e3552cf6-27c7-4100-942b-c5877c673046, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } } } }, OmicronZoneConfig { id: 9a0125d1-b636-4039-9e54-dea22726a38a, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } } } }, OmicronZoneConfig { id: 7c851202-bc92-4a32-b946-68379ad242d7, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 653e8bac-3f5b-49fe-a9f9-e9f0546059e9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External } } } }, OmicronZoneConfig { id: e4ffe9aa-395e-4536-a4d0-5cf0bc814584, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 48ce280c-ebea-49f3-9829-dc9b4d018a02, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: c2791662-4c87-4a63-a67a-4d6489fb4eaf, kind: Service { id: 48ce280c-ebea-49f3-9829-dc9b4d018a02 }, name: Name(\"external-dns-48ce280c-ebea-49f3-9829-dc9b4d018a02\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 86])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 38cec1c6-d097-4604-b9cb-30a5fab57e35, kind: Service { id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94 }, name: Name(\"external-dns-9f29eb53-b3d5-4fc1-b4d8-a252757e0c94\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 215])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 67179f00-d639-4f5b-b3c6-71d676113139, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: a724cf8c-c466-4981-aa80-a17ab2349f83, kind: Service { id: 67179f00-d639-4f5b-b3c6-71d676113139 }, name: Name(\"nexus-67179f00-d639-4f5b-b3c6-71d676113139\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 55])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7e2a6156-04b4-4797-8bed-d0c5c9cbe752, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: e31530f0-72e9-4544-bfbb-847c405d46c8, kind: Service { id: 7e2a6156-04b4-4797-8bed-d0c5c9cbe752 }, name: Name(\"nexus-7e2a6156-04b4-4797-8bed-d0c5c9cbe752\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 20a0479c-19b7-40a0-a935-5ff5399f7d02, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 79c95d4d-cff5-4419-8f3e-a6e216736b73, kind: Service { id: 20a0479c-19b7-40a0-a935-5ff5399f7d02 }, name: Name(\"nexus-20a0479c-19b7-40a0-a935-5ff5399f7d02\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 249])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: e42e8525-00d2-488f-a732-5c9514a5c7b4, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 96bb9301-f50f-4259-8a1b-169ec98cbff3, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } } } }, OmicronZoneConfig { id: 966bee63-3642-4e7a-a159-17eb59b72717, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: da8be60e-c115-4289-8526-a1ad7b366b93, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: d72762b4-6a43-4e1c-92b1-b432c1731075, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: c3d43ade-9569-403a-a6cc-494ec947793f, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } } } }, OmicronZoneConfig { id: f09c8d14-475d-49d8-b03f-d3a0b7b880cc, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } } } }, OmicronZoneConfig { id: b202e971-de3b-4a73-a534-ee56c23e79a3, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2b78106a-b661-444a-a40f-011ae4f0911a, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External } } } }, OmicronZoneConfig { id: f64b0db4-4e9f-4666-b267-c9abe9f8b189, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 56d79ac6-bb86-49e7-905e-343e52dbbb90, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd06e234-5970-4365-a0e5-6cd29811daea (zpool), kind: External } } } }, OmicronZoneConfig { id: 010164b7-56ba-4dda-8f67-3d5e05751e2e, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3488ba32-8748-4793-9389-5ea882fa963c, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool), kind: External } } } }, OmicronZoneConfig { id: 89e132b1-d133-478e-a8f7-cf9f30b211c9, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:15.209987363Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:15.210299783Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:15.210308005Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9f65db5e-d949-4342-9d90-6d691adea120, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:15.229775329Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:15.229820686Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9f65db5e-d949-4342-9d90-6d691adea120, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:15.249161547Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:15.249202067Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e3552cf6-27c7-4100-942b-c5877c673046, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:15.270663897Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:15.270712217Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e3552cf6-27c7-4100-942b-c5877c673046, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a0125d1-b636-4039-9e54-dea22726a38a, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:15.288114011Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:15.28814914Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a0125d1-b636-4039-9e54-dea22726a38a, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c851202-bc92-4a32-b946-68379ad242d7, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:15.306727428Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:15.306778644Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7c851202-bc92-4a32-b946-68379ad242d7, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 653e8bac-3f5b-49fe-a9f9-e9f0546059e9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:15.325141588Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:15.325178555Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 653e8bac-3f5b-49fe-a9f9-e9f0546059e9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e4ffe9aa-395e-4536-a4d0-5cf0bc814584, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:15.345775249Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:15.345811709Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e4ffe9aa-395e-4536-a4d0-5cf0bc814584, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48ce280c-ebea-49f3-9829-dc9b4d018a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:15.364909606Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:15.364948841Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 48ce280c-ebea-49f3-9829-dc9b4d018a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:15.384148279Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:15.384187149Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 96bb9301-f50f-4259-8a1b-169ec98cbff3, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:15.407294919Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:15.407337398Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 96bb9301-f50f-4259-8a1b-169ec98cbff3, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c3d43ade-9569-403a-a6cc-494ec947793f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:15.426062057Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:15.426100327Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c3d43ade-9569-403a-a6cc-494ec947793f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f09c8d14-475d-49d8-b03f-d3a0b7b880cc, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:15.446392276Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:15.446432918Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f09c8d14-475d-49d8-b03f-d3a0b7b880cc, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b202e971-de3b-4a73-a534-ee56c23e79a3, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:15.465975026Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:15.466013296Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b202e971-de3b-4a73-a534-ee56c23e79a3, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b78106a-b661-444a-a40f-011ae4f0911a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:15.485835768Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:15.485871629Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2b78106a-b661-444a-a40f-011ae4f0911a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f64b0db4-4e9f-4666-b267-c9abe9f8b189, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:15.50837737Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:15.508414431Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f64b0db4-4e9f-4666-b267-c9abe9f8b189, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56d79ac6-bb86-49e7-905e-343e52dbbb90, dataset_name: DatasetName { pool_name: ZpoolName { id: bd06e234-5970-4365-a0e5-6cd29811daea (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:15.52940121Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:15.52944773Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 56d79ac6-bb86-49e7-905e-343e52dbbb90, dataset_name: DatasetName { pool_name: ZpoolName { id: bd06e234-5970-4365-a0e5-6cd29811daea (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 010164b7-56ba-4dda-8f67-3d5e05751e2e, dataset_name: DatasetName { pool_name: ZpoolName { id: e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:15.550526758Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:15.550564363Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 010164b7-56ba-4dda-8f67-3d5e05751e2e, dataset_name: DatasetName { pool_name: ZpoolName { id: e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3488ba32-8748-4793-9389-5ea882fa963c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:15.571811834Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:15.571848941Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3488ba32-8748-4793-9389-5ea882fa963c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89e132b1-d133-478e-a8f7-cf9f30b211c9, dataset_name: DatasetName { pool_name: ZpoolName { id: cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:15.591370835Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:15.59141607Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 89e132b1-d133-478e-a8f7-cf9f30b211c9, dataset_name: DatasetName { pool_name: ZpoolName { id: cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:15.611501889Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/968bbfe8-8122-4a85-96dd-954188a2f89d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:15.611554409Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/e8396e20-0493-4030-b55b-5c9e59a237cf/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:15.611737573Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:15.611849381Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:15.705781702Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:15.705824622Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:15.705830594Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:15.705834597Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:15.705838554Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:15.705842482Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:15.705845463Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:15.705848492Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:15.705852411Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:15.705855504Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:15.705874639Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:15.705879805Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:15.705884032Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:15.705888974Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:15.705892921Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:15.70589683Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_f09c8d14-475d-49d8-b03f-d3a0b7b880cc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:15.705901012Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_b202e971-de3b-4a73-a534-ee56c23e79a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:15.705904893Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:15.705909646Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:15.70598942Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"9f70675d-49d9-4e16-bbd1-c72e25caac68","remote_addr":"[fd00:1122:3344:101::1]:39162","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a\", TimeNotSynchronized), (\"oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e\", TimeNotSynchronized), (\"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f\", TimeNotSynchronized), (\"oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189\", TimeNotSynchronized), (\"oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90\", TimeNotSynchronized), (\"oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c\", TimeNotSynchronized), (\"oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9\", TimeNotSynchronized), (\"oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02\", TimeNotSynchronized), (\"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752\", TimeNotSynchronized), (\"oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94\", TimeNotSynchronized), (\"oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02\", TimeNotSynchronized), (\"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3\", TimeNotSynchronized), (\"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717\", TimeNotSynchronized), (\"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93\", TimeNotSynchronized), (\"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075\", TimeNotSynchronized), (\"oxz_crucible_f09c8d14-475d-49d8-b03f-d3a0b7b880cc\", TimeNotSynchronized), (\"oxz_crucible_b202e971-de3b-4a73-a534-ee56c23e79a3\", TimeNotSynchronized), (\"oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4\", TimeNotSynchronized), (\"oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139\", TimeNotSynchronized)]","latency_us":494175,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:15.706177759Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"1.765579532s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"9f70675d-49d9-4e16-bbd1-c72e25caac68\", \"content-length\": \"124\", \"date\": \"Wed, 26 Jun 2024 05:36:15 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"9f70675d-49d9-4e16-bbd1-c72e25caac68\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 9f65db5e-d949-4342-9d90-6d691adea120, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: a4d358ae-9d04-40a4-b4a5-ac8418ace88d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 707beb4d-acae-4ff1-bb7e-97992526c5fb, kind: Service { id: a4d358ae-9d04-40a4-b4a5-ac8418ace88d }, name: Name(\"ntp-a4d358ae-9d04-40a4-b4a5-ac8418ace88d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 238, 27])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e3552cf6-27c7-4100-942b-c5877c673046, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } } } }, OmicronZoneConfig { id: 9a0125d1-b636-4039-9e54-dea22726a38a, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } } } }, OmicronZoneConfig { id: 7c851202-bc92-4a32-b946-68379ad242d7, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 653e8bac-3f5b-49fe-a9f9-e9f0546059e9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External } } } }, OmicronZoneConfig { id: e4ffe9aa-395e-4536-a4d0-5cf0bc814584, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 48ce280c-ebea-49f3-9829-dc9b4d018a02, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: c2791662-4c87-4a63-a67a-4d6489fb4eaf, kind: Service { id: 48ce280c-ebea-49f3-9829-dc9b4d018a02 }, name: Name(\"external-dns-48ce280c-ebea-49f3-9829-dc9b4d018a02\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 86])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 38cec1c6-d097-4604-b9cb-30a5fab57e35, kind: Service { id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94 }, name: Name(\"external-dns-9f29eb53-b3d5-4fc1-b4d8-a252757e0c94\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 215])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 67179f00-d639-4f5b-b3c6-71d676113139, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: a724cf8c-c466-4981-aa80-a17ab2349f83, kind: Service { id: 67179f00-d639-4f5b-b3c6-71d676113139 }, name: Name(\"nexus-67179f00-d639-4f5b-b3c6-71d676113139\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 55])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7e2a6156-04b4-4797-8bed-d0c5c9cbe752, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: e31530f0-72e9-4544-bfbb-847c405d46c8, kind: Service { id: 7e2a6156-04b4-4797-8bed-d0c5c9cbe752 }, name: Name(\"nexus-7e2a6156-04b4-4797-8bed-d0c5c9cbe752\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 20a0479c-19b7-40a0-a935-5ff5399f7d02, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 79c95d4d-cff5-4419-8f3e-a6e216736b73, kind: Service { id: 20a0479c-19b7-40a0-a935-5ff5399f7d02 }, name: Name(\"nexus-20a0479c-19b7-40a0-a935-5ff5399f7d02\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 249])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: e42e8525-00d2-488f-a732-5c9514a5c7b4, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 96bb9301-f50f-4259-8a1b-169ec98cbff3, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } } } }, OmicronZoneConfig { id: 966bee63-3642-4e7a-a159-17eb59b72717, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: da8be60e-c115-4289-8526-a1ad7b366b93, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: d72762b4-6a43-4e1c-92b1-b432c1731075, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: c3d43ade-9569-403a-a6cc-494ec947793f, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } } } }, OmicronZoneConfig { id: f09c8d14-475d-49d8-b03f-d3a0b7b880cc, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } } } }, OmicronZoneConfig { id: b202e971-de3b-4a73-a534-ee56c23e79a3, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2b78106a-b661-444a-a40f-011ae4f0911a, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External } } } }, OmicronZoneConfig { id: f64b0db4-4e9f-4666-b267-c9abe9f8b189, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 56d79ac6-bb86-49e7-905e-343e52dbbb90, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd06e234-5970-4365-a0e5-6cd29811daea (zpool), kind: External } } } }, OmicronZoneConfig { id: 010164b7-56ba-4dda-8f67-3d5e05751e2e, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3488ba32-8748-4793-9389-5ea882fa963c, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool), kind: External } } } }, OmicronZoneConfig { id: 89e132b1-d133-478e-a8f7-cf9f30b211c9, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:17.478905088Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:17.479218707Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:17.479231026Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9f65db5e-d949-4342-9d90-6d691adea120, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:17.501378859Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:17.50141097Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9f65db5e-d949-4342-9d90-6d691adea120, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:17.521240379Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:17.521273886Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e3552cf6-27c7-4100-942b-c5877c673046, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:17.542059593Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:17.542098556Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e3552cf6-27c7-4100-942b-c5877c673046, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a0125d1-b636-4039-9e54-dea22726a38a, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:17.561636597Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:17.561666054Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a0125d1-b636-4039-9e54-dea22726a38a, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c851202-bc92-4a32-b946-68379ad242d7, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:17.582898964Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:17.58293729Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7c851202-bc92-4a32-b946-68379ad242d7, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 653e8bac-3f5b-49fe-a9f9-e9f0546059e9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:17.601968485Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:17.601999517Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 653e8bac-3f5b-49fe-a9f9-e9f0546059e9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e4ffe9aa-395e-4536-a4d0-5cf0bc814584, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:17.620790122Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:17.620819316Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e4ffe9aa-395e-4536-a4d0-5cf0bc814584, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48ce280c-ebea-49f3-9829-dc9b4d018a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:17.638472304Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:17.638497823Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 48ce280c-ebea-49f3-9829-dc9b4d018a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:17.658315041Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:17.658346551Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 96bb9301-f50f-4259-8a1b-169ec98cbff3, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:17.678395195Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:17.678422037Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 96bb9301-f50f-4259-8a1b-169ec98cbff3, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c3d43ade-9569-403a-a6cc-494ec947793f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:17.698188571Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:17.698222744Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c3d43ade-9569-403a-a6cc-494ec947793f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f09c8d14-475d-49d8-b03f-d3a0b7b880cc, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:17.716358234Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:17.716391572Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f09c8d14-475d-49d8-b03f-d3a0b7b880cc, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b202e971-de3b-4a73-a534-ee56c23e79a3, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:17.734820496Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:17.734851715Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b202e971-de3b-4a73-a534-ee56c23e79a3, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b78106a-b661-444a-a40f-011ae4f0911a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:17.753014025Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:17.753051076Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2b78106a-b661-444a-a40f-011ae4f0911a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f64b0db4-4e9f-4666-b267-c9abe9f8b189, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:17.772145343Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:17.772174913Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f64b0db4-4e9f-4666-b267-c9abe9f8b189, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56d79ac6-bb86-49e7-905e-343e52dbbb90, dataset_name: DatasetName { pool_name: ZpoolName { id: bd06e234-5970-4365-a0e5-6cd29811daea (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:17.794741227Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:17.794775906Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 56d79ac6-bb86-49e7-905e-343e52dbbb90, dataset_name: DatasetName { pool_name: ZpoolName { id: bd06e234-5970-4365-a0e5-6cd29811daea (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 010164b7-56ba-4dda-8f67-3d5e05751e2e, dataset_name: DatasetName { pool_name: ZpoolName { id: e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:17.81376342Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:17.813791977Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 010164b7-56ba-4dda-8f67-3d5e05751e2e, dataset_name: DatasetName { pool_name: ZpoolName { id: e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3488ba32-8748-4793-9389-5ea882fa963c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:17.833469647Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:17.833498063Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3488ba32-8748-4793-9389-5ea882fa963c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89e132b1-d133-478e-a8f7-cf9f30b211c9, dataset_name: DatasetName { pool_name: ZpoolName { id: cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:17.852796923Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:17.852847595Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 89e132b1-d133-478e-a8f7-cf9f30b211c9, dataset_name: DatasetName { pool_name: ZpoolName { id: cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:17.871386912Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/968bbfe8-8122-4a85-96dd-954188a2f89d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:17.871431763Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/e8396e20-0493-4030-b55b-5c9e59a237cf/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:17.871647891Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:17.871762587Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:17.968724568Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:17.968773441Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:17.968780838Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:17.968785816Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:17.968790682Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:17.96879685Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:17.968802344Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:17.968807126Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:17.968812161Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:17.968828736Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:17.968835758Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_b202e971-de3b-4a73-a534-ee56c23e79a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:17.968842424Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:17.968847824Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:17.968851987Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:17.968855915Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:17.968859365Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:17.968864624Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:17.968869799Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:17.968873193Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_f09c8d14-475d-49d8-b03f-d3a0b7b880cc"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:17.96894407Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"10e6de82-6cca-4ed5-acb6-5a15c0e52f5d","remote_addr":"[fd00:1122:3344:101::1]:39162","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90\", TimeNotSynchronized), (\"oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e\", TimeNotSynchronized), (\"oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94\", TimeNotSynchronized), (\"oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9\", TimeNotSynchronized), (\"oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02\", TimeNotSynchronized), (\"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717\", TimeNotSynchronized), (\"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93\", TimeNotSynchronized), (\"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752\", TimeNotSynchronized), (\"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f\", TimeNotSynchronized), (\"oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139\", TimeNotSynchronized), (\"oxz_crucible_b202e971-de3b-4a73-a534-ee56c23e79a3\", TimeNotSynchronized), (\"oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189\", TimeNotSynchronized), (\"oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02\", TimeNotSynchronized), (\"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3\", TimeNotSynchronized), (\"oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4\", TimeNotSynchronized), (\"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075\", TimeNotSynchronized), (\"oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c\", TimeNotSynchronized), (\"oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a\", TimeNotSynchronized), (\"oxz_crucible_f09c8d14-475d-49d8-b03f-d3a0b7b880cc\", TimeNotSynchronized)]","latency_us":488212,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:17.969152828Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"1.705624013s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"10e6de82-6cca-4ed5-acb6-5a15c0e52f5d\", \"content-length\": \"124\", \"date\": \"Wed, 26 Jun 2024 05:36:17 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"10e6de82-6cca-4ed5-acb6-5a15c0e52f5d\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 9f65db5e-d949-4342-9d90-6d691adea120, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: a4d358ae-9d04-40a4-b4a5-ac8418ace88d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 707beb4d-acae-4ff1-bb7e-97992526c5fb, kind: Service { id: a4d358ae-9d04-40a4-b4a5-ac8418ace88d }, name: Name(\"ntp-a4d358ae-9d04-40a4-b4a5-ac8418ace88d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 238, 27])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e3552cf6-27c7-4100-942b-c5877c673046, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } } } }, OmicronZoneConfig { id: 9a0125d1-b636-4039-9e54-dea22726a38a, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } } } }, OmicronZoneConfig { id: 7c851202-bc92-4a32-b946-68379ad242d7, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 653e8bac-3f5b-49fe-a9f9-e9f0546059e9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External } } } }, OmicronZoneConfig { id: e4ffe9aa-395e-4536-a4d0-5cf0bc814584, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 48ce280c-ebea-49f3-9829-dc9b4d018a02, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: c2791662-4c87-4a63-a67a-4d6489fb4eaf, kind: Service { id: 48ce280c-ebea-49f3-9829-dc9b4d018a02 }, name: Name(\"external-dns-48ce280c-ebea-49f3-9829-dc9b4d018a02\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 86])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 38cec1c6-d097-4604-b9cb-30a5fab57e35, kind: Service { id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94 }, name: Name(\"external-dns-9f29eb53-b3d5-4fc1-b4d8-a252757e0c94\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 215])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 67179f00-d639-4f5b-b3c6-71d676113139, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: a724cf8c-c466-4981-aa80-a17ab2349f83, kind: Service { id: 67179f00-d639-4f5b-b3c6-71d676113139 }, name: Name(\"nexus-67179f00-d639-4f5b-b3c6-71d676113139\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 55])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7e2a6156-04b4-4797-8bed-d0c5c9cbe752, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: e31530f0-72e9-4544-bfbb-847c405d46c8, kind: Service { id: 7e2a6156-04b4-4797-8bed-d0c5c9cbe752 }, name: Name(\"nexus-7e2a6156-04b4-4797-8bed-d0c5c9cbe752\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 20a0479c-19b7-40a0-a935-5ff5399f7d02, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 79c95d4d-cff5-4419-8f3e-a6e216736b73, kind: Service { id: 20a0479c-19b7-40a0-a935-5ff5399f7d02 }, name: Name(\"nexus-20a0479c-19b7-40a0-a935-5ff5399f7d02\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 249])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: e42e8525-00d2-488f-a732-5c9514a5c7b4, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 96bb9301-f50f-4259-8a1b-169ec98cbff3, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } } } }, OmicronZoneConfig { id: 966bee63-3642-4e7a-a159-17eb59b72717, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: da8be60e-c115-4289-8526-a1ad7b366b93, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: d72762b4-6a43-4e1c-92b1-b432c1731075, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: c3d43ade-9569-403a-a6cc-494ec947793f, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } } } }, OmicronZoneConfig { id: f09c8d14-475d-49d8-b03f-d3a0b7b880cc, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } } } }, OmicronZoneConfig { id: b202e971-de3b-4a73-a534-ee56c23e79a3, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2b78106a-b661-444a-a40f-011ae4f0911a, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External } } } }, OmicronZoneConfig { id: f64b0db4-4e9f-4666-b267-c9abe9f8b189, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 56d79ac6-bb86-49e7-905e-343e52dbbb90, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd06e234-5970-4365-a0e5-6cd29811daea (zpool), kind: External } } } }, OmicronZoneConfig { id: 010164b7-56ba-4dda-8f67-3d5e05751e2e, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3488ba32-8748-4793-9389-5ea882fa963c, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool), kind: External } } } }, OmicronZoneConfig { id: 89e132b1-d133-478e-a8f7-cf9f30b211c9, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:19.682267677Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:19.68330286Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:19.683320494Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9f65db5e-d949-4342-9d90-6d691adea120, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:19.705213535Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:19.705256699Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9f65db5e-d949-4342-9d90-6d691adea120, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:19.725231912Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:19.725274006Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e3552cf6-27c7-4100-942b-c5877c673046, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:19.74456112Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:19.744601246Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e3552cf6-27c7-4100-942b-c5877c673046, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a0125d1-b636-4039-9e54-dea22726a38a, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:19.765365732Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:19.76540626Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a0125d1-b636-4039-9e54-dea22726a38a, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c851202-bc92-4a32-b946-68379ad242d7, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:19.784355556Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:19.78440741Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7c851202-bc92-4a32-b946-68379ad242d7, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 653e8bac-3f5b-49fe-a9f9-e9f0546059e9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:19.803429976Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:19.80347299Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 653e8bac-3f5b-49fe-a9f9-e9f0546059e9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e4ffe9aa-395e-4536-a4d0-5cf0bc814584, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:19.822790586Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:19.82283374Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e4ffe9aa-395e-4536-a4d0-5cf0bc814584, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48ce280c-ebea-49f3-9829-dc9b4d018a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:19.841430916Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:19.84147017Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 48ce280c-ebea-49f3-9829-dc9b4d018a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:19.861098037Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:19.861145035Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 96bb9301-f50f-4259-8a1b-169ec98cbff3, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:19.880893858Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:19.880937218Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 96bb9301-f50f-4259-8a1b-169ec98cbff3, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c3d43ade-9569-403a-a6cc-494ec947793f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:19.90327043Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:19.903314597Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c3d43ade-9569-403a-a6cc-494ec947793f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f09c8d14-475d-49d8-b03f-d3a0b7b880cc, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:19.924318047Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:19.924360245Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f09c8d14-475d-49d8-b03f-d3a0b7b880cc, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b202e971-de3b-4a73-a534-ee56c23e79a3, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:19.944129178Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:19.944175895Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b202e971-de3b-4a73-a534-ee56c23e79a3, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b78106a-b661-444a-a40f-011ae4f0911a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:19.963717745Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:19.963771371Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2b78106a-b661-444a-a40f-011ae4f0911a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f64b0db4-4e9f-4666-b267-c9abe9f8b189, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:19.983216562Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:19.983256923Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f64b0db4-4e9f-4666-b267-c9abe9f8b189, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56d79ac6-bb86-49e7-905e-343e52dbbb90, dataset_name: DatasetName { pool_name: ZpoolName { id: bd06e234-5970-4365-a0e5-6cd29811daea (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:20.003231185Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:20.003278099Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 56d79ac6-bb86-49e7-905e-343e52dbbb90, dataset_name: DatasetName { pool_name: ZpoolName { id: bd06e234-5970-4365-a0e5-6cd29811daea (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 010164b7-56ba-4dda-8f67-3d5e05751e2e, dataset_name: DatasetName { pool_name: ZpoolName { id: e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:20.025023768Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:20.025074292Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 010164b7-56ba-4dda-8f67-3d5e05751e2e, dataset_name: DatasetName { pool_name: ZpoolName { id: e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3488ba32-8748-4793-9389-5ea882fa963c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:20.043493671Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:20.04354275Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3488ba32-8748-4793-9389-5ea882fa963c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89e132b1-d133-478e-a8f7-cf9f30b211c9, dataset_name: DatasetName { pool_name: ZpoolName { id: cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:20.063837212Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:20.063899098Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 89e132b1-d133-478e-a8f7-cf9f30b211c9, dataset_name: DatasetName { pool_name: ZpoolName { id: cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:20.085433439Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/968bbfe8-8122-4a85-96dd-954188a2f89d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:20.085488519Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/e8396e20-0493-4030-b55b-5c9e59a237cf/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:20.085673557Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:20.085995997Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:20.186871883Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:20.186930731Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:20.186936938Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:20.186941663Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:20.186945881Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:20.186950944Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:20.186958856Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:20.186962925Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:20.186966122Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_b202e971-de3b-4a73-a534-ee56c23e79a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:20.186981854Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:20.186985173Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_f09c8d14-475d-49d8-b03f-d3a0b7b880cc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:20.186990142Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:20.186993957Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:20.18699812Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:20.187002414Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:20.187005517Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:20.187008892Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:20.187012886Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:20.187016768Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:20.187043927Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"b84d4c76-c65c-41b8-b756-e46aa4016e84","remote_addr":"[fd00:1122:3344:101::1]:39162","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139\", TimeNotSynchronized), (\"oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9\", TimeNotSynchronized), (\"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075\", TimeNotSynchronized), (\"oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94\", TimeNotSynchronized), (\"oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02\", TimeNotSynchronized), (\"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717\", TimeNotSynchronized), (\"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93\", TimeNotSynchronized), (\"oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02\", TimeNotSynchronized), (\"oxz_crucible_b202e971-de3b-4a73-a534-ee56c23e79a3\", TimeNotSynchronized), (\"oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90\", TimeNotSynchronized), (\"oxz_crucible_f09c8d14-475d-49d8-b03f-d3a0b7b880cc\", TimeNotSynchronized), (\"oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e\", TimeNotSynchronized), (\"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3\", TimeNotSynchronized), (\"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752\", TimeNotSynchronized), (\"oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a\", TimeNotSynchronized), (\"oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c\", TimeNotSynchronized), (\"oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189\", TimeNotSynchronized), (\"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f\", TimeNotSynchronized), (\"oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4\", TimeNotSynchronized)]","latency_us":502879,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:20.187296341Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"3.135069395s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b84d4c76-c65c-41b8-b756-e46aa4016e84\", \"content-length\": \"124\", \"date\": \"Wed, 26 Jun 2024 05:36:19 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b84d4c76-c65c-41b8-b756-e46aa4016e84\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 9f65db5e-d949-4342-9d90-6d691adea120, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: a4d358ae-9d04-40a4-b4a5-ac8418ace88d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 707beb4d-acae-4ff1-bb7e-97992526c5fb, kind: Service { id: a4d358ae-9d04-40a4-b4a5-ac8418ace88d }, name: Name(\"ntp-a4d358ae-9d04-40a4-b4a5-ac8418ace88d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 238, 27])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e3552cf6-27c7-4100-942b-c5877c673046, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } } } }, OmicronZoneConfig { id: 9a0125d1-b636-4039-9e54-dea22726a38a, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } } } }, OmicronZoneConfig { id: 7c851202-bc92-4a32-b946-68379ad242d7, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 653e8bac-3f5b-49fe-a9f9-e9f0546059e9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External } } } }, OmicronZoneConfig { id: e4ffe9aa-395e-4536-a4d0-5cf0bc814584, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 48ce280c-ebea-49f3-9829-dc9b4d018a02, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: c2791662-4c87-4a63-a67a-4d6489fb4eaf, kind: Service { id: 48ce280c-ebea-49f3-9829-dc9b4d018a02 }, name: Name(\"external-dns-48ce280c-ebea-49f3-9829-dc9b4d018a02\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 86])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 38cec1c6-d097-4604-b9cb-30a5fab57e35, kind: Service { id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94 }, name: Name(\"external-dns-9f29eb53-b3d5-4fc1-b4d8-a252757e0c94\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 215])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 67179f00-d639-4f5b-b3c6-71d676113139, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: a724cf8c-c466-4981-aa80-a17ab2349f83, kind: Service { id: 67179f00-d639-4f5b-b3c6-71d676113139 }, name: Name(\"nexus-67179f00-d639-4f5b-b3c6-71d676113139\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 55])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7e2a6156-04b4-4797-8bed-d0c5c9cbe752, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: e31530f0-72e9-4544-bfbb-847c405d46c8, kind: Service { id: 7e2a6156-04b4-4797-8bed-d0c5c9cbe752 }, name: Name(\"nexus-7e2a6156-04b4-4797-8bed-d0c5c9cbe752\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 20a0479c-19b7-40a0-a935-5ff5399f7d02, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 79c95d4d-cff5-4419-8f3e-a6e216736b73, kind: Service { id: 20a0479c-19b7-40a0-a935-5ff5399f7d02 }, name: Name(\"nexus-20a0479c-19b7-40a0-a935-5ff5399f7d02\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 249])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: e42e8525-00d2-488f-a732-5c9514a5c7b4, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 96bb9301-f50f-4259-8a1b-169ec98cbff3, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } } } }, OmicronZoneConfig { id: 966bee63-3642-4e7a-a159-17eb59b72717, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: da8be60e-c115-4289-8526-a1ad7b366b93, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: d72762b4-6a43-4e1c-92b1-b432c1731075, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: c3d43ade-9569-403a-a6cc-494ec947793f, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } } } }, OmicronZoneConfig { id: f09c8d14-475d-49d8-b03f-d3a0b7b880cc, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } } } }, OmicronZoneConfig { id: b202e971-de3b-4a73-a534-ee56c23e79a3, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2b78106a-b661-444a-a40f-011ae4f0911a, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External } } } }, OmicronZoneConfig { id: f64b0db4-4e9f-4666-b267-c9abe9f8b189, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 56d79ac6-bb86-49e7-905e-343e52dbbb90, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd06e234-5970-4365-a0e5-6cd29811daea (zpool), kind: External } } } }, OmicronZoneConfig { id: 010164b7-56ba-4dda-8f67-3d5e05751e2e, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3488ba32-8748-4793-9389-5ea882fa963c, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool), kind: External } } } }, OmicronZoneConfig { id: 89e132b1-d133-478e-a8f7-cf9f30b211c9, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:23.335322379Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:23.335616554Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:23.335628535Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9f65db5e-d949-4342-9d90-6d691adea120, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:23.355791655Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:23.355827721Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9f65db5e-d949-4342-9d90-6d691adea120, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:23.376431523Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:23.37647284Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e3552cf6-27c7-4100-942b-c5877c673046, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:23.394571169Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:23.394600354Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e3552cf6-27c7-4100-942b-c5877c673046, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a0125d1-b636-4039-9e54-dea22726a38a, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:23.412231947Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:23.412264779Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a0125d1-b636-4039-9e54-dea22726a38a, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c851202-bc92-4a32-b946-68379ad242d7, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:23.431793945Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:23.431826243Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7c851202-bc92-4a32-b946-68379ad242d7, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 653e8bac-3f5b-49fe-a9f9-e9f0546059e9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:23.450647238Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:23.450680492Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 653e8bac-3f5b-49fe-a9f9-e9f0546059e9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e4ffe9aa-395e-4536-a4d0-5cf0bc814584, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:23.469800643Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:23.46983461Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e4ffe9aa-395e-4536-a4d0-5cf0bc814584, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48ce280c-ebea-49f3-9829-dc9b4d018a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:23.489199082Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:23.489236133Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 48ce280c-ebea-49f3-9829-dc9b4d018a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:23.509598172Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:23.509628567Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 96bb9301-f50f-4259-8a1b-169ec98cbff3, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:23.529158247Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:23.529191117Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 96bb9301-f50f-4259-8a1b-169ec98cbff3, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c3d43ade-9569-403a-a6cc-494ec947793f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:23.548151137Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:23.548183969Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c3d43ade-9569-403a-a6cc-494ec947793f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f09c8d14-475d-49d8-b03f-d3a0b7b880cc, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:23.566836761Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:23.566936434Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f09c8d14-475d-49d8-b03f-d3a0b7b880cc, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b202e971-de3b-4a73-a534-ee56c23e79a3, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:23.58471378Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:23.584751037Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b202e971-de3b-4a73-a534-ee56c23e79a3, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b78106a-b661-444a-a40f-011ae4f0911a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:23.602659372Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:23.60269422Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2b78106a-b661-444a-a40f-011ae4f0911a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f64b0db4-4e9f-4666-b267-c9abe9f8b189, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:23.621674651Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:23.621711289Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f64b0db4-4e9f-4666-b267-c9abe9f8b189, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56d79ac6-bb86-49e7-905e-343e52dbbb90, dataset_name: DatasetName { pool_name: ZpoolName { id: bd06e234-5970-4365-a0e5-6cd29811daea (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:23.640463137Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:23.64049611Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 56d79ac6-bb86-49e7-905e-343e52dbbb90, dataset_name: DatasetName { pool_name: ZpoolName { id: bd06e234-5970-4365-a0e5-6cd29811daea (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 010164b7-56ba-4dda-8f67-3d5e05751e2e, dataset_name: DatasetName { pool_name: ZpoolName { id: e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:23.660085684Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:23.660124572Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 010164b7-56ba-4dda-8f67-3d5e05751e2e, dataset_name: DatasetName { pool_name: ZpoolName { id: e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3488ba32-8748-4793-9389-5ea882fa963c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:23.681186207Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:23.681220013Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3488ba32-8748-4793-9389-5ea882fa963c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89e132b1-d133-478e-a8f7-cf9f30b211c9, dataset_name: DatasetName { pool_name: ZpoolName { id: cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:23.70045251Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:23.700487864Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 89e132b1-d133-478e-a8f7-cf9f30b211c9, dataset_name: DatasetName { pool_name: ZpoolName { id: cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:23.719223542Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/968bbfe8-8122-4a85-96dd-954188a2f89d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:23.719271309Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/e8396e20-0493-4030-b55b-5c9e59a237cf/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:23.719545668Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:23.719678515Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:23.81142622Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:23.811485088Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:23.811496019Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:23.811504739Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:23.81151072Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:23.811518614Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:23.811524773Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:23.811532667Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:23.811540317Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_b202e971-de3b-4a73-a534-ee56c23e79a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:23.811556114Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:23.811561055Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_f09c8d14-475d-49d8-b03f-d3a0b7b880cc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:23.811565471Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:23.811569839Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:23.811575943Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:23.811583743Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:23.811590203Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:23.811594628Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:23.811599334Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:23.811606881Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:23.811630562Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"77706e08-1b11-4d30-ac78-cc4a7cfec8f7","remote_addr":"[fd00:1122:3344:101::1]:39162","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075\", TimeNotSynchronized), (\"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752\", TimeNotSynchronized), (\"oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4\", TimeNotSynchronized), (\"oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94\", TimeNotSynchronized), (\"oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c\", TimeNotSynchronized), (\"oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9\", TimeNotSynchronized), (\"oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02\", TimeNotSynchronized), (\"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3\", TimeNotSynchronized), (\"oxz_crucible_b202e971-de3b-4a73-a534-ee56c23e79a3\", TimeNotSynchronized), (\"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93\", TimeNotSynchronized), (\"oxz_crucible_f09c8d14-475d-49d8-b03f-d3a0b7b880cc\", TimeNotSynchronized), (\"oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a\", TimeNotSynchronized), (\"oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02\", TimeNotSynchronized), (\"oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139\", TimeNotSynchronized), (\"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717\", TimeNotSynchronized), (\"oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90\", TimeNotSynchronized), (\"oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e\", TimeNotSynchronized), (\"oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189\", TimeNotSynchronized), (\"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f\", TimeNotSynchronized)]","latency_us":474572,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:23.811899163Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"3.268246736s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"77706e08-1b11-4d30-ac78-cc4a7cfec8f7\", \"content-length\": \"124\", \"date\": \"Wed, 26 Jun 2024 05:36:23 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"77706e08-1b11-4d30-ac78-cc4a7cfec8f7\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 9f65db5e-d949-4342-9d90-6d691adea120, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: a4d358ae-9d04-40a4-b4a5-ac8418ace88d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 707beb4d-acae-4ff1-bb7e-97992526c5fb, kind: Service { id: a4d358ae-9d04-40a4-b4a5-ac8418ace88d }, name: Name(\"ntp-a4d358ae-9d04-40a4-b4a5-ac8418ace88d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 238, 27])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e3552cf6-27c7-4100-942b-c5877c673046, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } } } }, OmicronZoneConfig { id: 9a0125d1-b636-4039-9e54-dea22726a38a, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } } } }, OmicronZoneConfig { id: 7c851202-bc92-4a32-b946-68379ad242d7, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 653e8bac-3f5b-49fe-a9f9-e9f0546059e9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External } } } }, OmicronZoneConfig { id: e4ffe9aa-395e-4536-a4d0-5cf0bc814584, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 48ce280c-ebea-49f3-9829-dc9b4d018a02, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: c2791662-4c87-4a63-a67a-4d6489fb4eaf, kind: Service { id: 48ce280c-ebea-49f3-9829-dc9b4d018a02 }, name: Name(\"external-dns-48ce280c-ebea-49f3-9829-dc9b4d018a02\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 86])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 38cec1c6-d097-4604-b9cb-30a5fab57e35, kind: Service { id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94 }, name: Name(\"external-dns-9f29eb53-b3d5-4fc1-b4d8-a252757e0c94\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 215])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 67179f00-d639-4f5b-b3c6-71d676113139, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: a724cf8c-c466-4981-aa80-a17ab2349f83, kind: Service { id: 67179f00-d639-4f5b-b3c6-71d676113139 }, name: Name(\"nexus-67179f00-d639-4f5b-b3c6-71d676113139\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 55])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7e2a6156-04b4-4797-8bed-d0c5c9cbe752, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: e31530f0-72e9-4544-bfbb-847c405d46c8, kind: Service { id: 7e2a6156-04b4-4797-8bed-d0c5c9cbe752 }, name: Name(\"nexus-7e2a6156-04b4-4797-8bed-d0c5c9cbe752\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 20a0479c-19b7-40a0-a935-5ff5399f7d02, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 79c95d4d-cff5-4419-8f3e-a6e216736b73, kind: Service { id: 20a0479c-19b7-40a0-a935-5ff5399f7d02 }, name: Name(\"nexus-20a0479c-19b7-40a0-a935-5ff5399f7d02\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 249])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: e42e8525-00d2-488f-a732-5c9514a5c7b4, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 96bb9301-f50f-4259-8a1b-169ec98cbff3, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } } } }, OmicronZoneConfig { id: 966bee63-3642-4e7a-a159-17eb59b72717, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: da8be60e-c115-4289-8526-a1ad7b366b93, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: d72762b4-6a43-4e1c-92b1-b432c1731075, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: c3d43ade-9569-403a-a6cc-494ec947793f, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } } } }, OmicronZoneConfig { id: f09c8d14-475d-49d8-b03f-d3a0b7b880cc, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } } } }, OmicronZoneConfig { id: b202e971-de3b-4a73-a534-ee56c23e79a3, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2b78106a-b661-444a-a40f-011ae4f0911a, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External } } } }, OmicronZoneConfig { id: f64b0db4-4e9f-4666-b267-c9abe9f8b189, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 56d79ac6-bb86-49e7-905e-343e52dbbb90, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd06e234-5970-4365-a0e5-6cd29811daea (zpool), kind: External } } } }, OmicronZoneConfig { id: 010164b7-56ba-4dda-8f67-3d5e05751e2e, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3488ba32-8748-4793-9389-5ea882fa963c, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool), kind: External } } } }, OmicronZoneConfig { id: 89e132b1-d133-478e-a8f7-cf9f30b211c9, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:27.093224844Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:27.093552481Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:27.0935708Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9f65db5e-d949-4342-9d90-6d691adea120, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:27.11580065Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:27.115852016Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9f65db5e-d949-4342-9d90-6d691adea120, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:27.13680258Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:27.136862768Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e3552cf6-27c7-4100-942b-c5877c673046, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:27.156584604Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:27.156634527Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e3552cf6-27c7-4100-942b-c5877c673046, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a0125d1-b636-4039-9e54-dea22726a38a, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:27.175851261Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:27.175905769Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a0125d1-b636-4039-9e54-dea22726a38a, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c851202-bc92-4a32-b946-68379ad242d7, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:27.193513613Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:27.193572133Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7c851202-bc92-4a32-b946-68379ad242d7, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 653e8bac-3f5b-49fe-a9f9-e9f0546059e9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:27.21434316Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:27.214394733Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 653e8bac-3f5b-49fe-a9f9-e9f0546059e9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e4ffe9aa-395e-4536-a4d0-5cf0bc814584, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:27.234311402Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:27.234365478Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e4ffe9aa-395e-4536-a4d0-5cf0bc814584, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48ce280c-ebea-49f3-9829-dc9b4d018a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:27.253087874Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:27.253144734Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 48ce280c-ebea-49f3-9829-dc9b4d018a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:27.272347163Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:27.272410118Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 96bb9301-f50f-4259-8a1b-169ec98cbff3, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:27.291187098Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:27.291237106Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 96bb9301-f50f-4259-8a1b-169ec98cbff3, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c3d43ade-9569-403a-a6cc-494ec947793f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:27.310071181Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:27.310130601Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c3d43ade-9569-403a-a6cc-494ec947793f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f09c8d14-475d-49d8-b03f-d3a0b7b880cc, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:27.328058457Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:27.32810102Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f09c8d14-475d-49d8-b03f-d3a0b7b880cc, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b202e971-de3b-4a73-a534-ee56c23e79a3, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:27.34635477Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:27.346407843Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b202e971-de3b-4a73-a534-ee56c23e79a3, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b78106a-b661-444a-a40f-011ae4f0911a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:27.36539013Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:27.365453095Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2b78106a-b661-444a-a40f-011ae4f0911a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f64b0db4-4e9f-4666-b267-c9abe9f8b189, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:27.38360725Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:27.383658176Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f64b0db4-4e9f-4666-b267-c9abe9f8b189, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56d79ac6-bb86-49e7-905e-343e52dbbb90, dataset_name: DatasetName { pool_name: ZpoolName { id: bd06e234-5970-4365-a0e5-6cd29811daea (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:27.402236648Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:27.402294119Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 56d79ac6-bb86-49e7-905e-343e52dbbb90, dataset_name: DatasetName { pool_name: ZpoolName { id: bd06e234-5970-4365-a0e5-6cd29811daea (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 010164b7-56ba-4dda-8f67-3d5e05751e2e, dataset_name: DatasetName { pool_name: ZpoolName { id: e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:27.420516023Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:27.420567492Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 010164b7-56ba-4dda-8f67-3d5e05751e2e, dataset_name: DatasetName { pool_name: ZpoolName { id: e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3488ba32-8748-4793-9389-5ea882fa963c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:27.438337874Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:27.43838626Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3488ba32-8748-4793-9389-5ea882fa963c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89e132b1-d133-478e-a8f7-cf9f30b211c9, dataset_name: DatasetName { pool_name: ZpoolName { id: cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:27.456785437Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:27.456836494Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 89e132b1-d133-478e-a8f7-cf9f30b211c9, dataset_name: DatasetName { pool_name: ZpoolName { id: cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:27.475183547Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/968bbfe8-8122-4a85-96dd-954188a2f89d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:27.475251237Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/e8396e20-0493-4030-b55b-5c9e59a237cf/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:27.475525161Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:27.475655645Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:27.567876512Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_f09c8d14-475d-49d8-b03f-d3a0b7b880cc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:27.567913065Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:27.567918371Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:27.567922985Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_b202e971-de3b-4a73-a534-ee56c23e79a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:27.567927091Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:27.567932135Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:27.567936297Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:27.567945166Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:27.567967817Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:27.567973095Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:27.567978298Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:27.567982536Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:27.567985789Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:27.567990064Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:27.567993186Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:27.567996365Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:27.567999692Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:27.568003743Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:27.568007371Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:27.568059075Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"8407801c-85d6-4354-82bf-e7d6eb7f2994","remote_addr":"[fd00:1122:3344:101::1]:39162","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_f09c8d14-475d-49d8-b03f-d3a0b7b880cc\", TimeNotSynchronized), (\"oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a\", TimeNotSynchronized), (\"oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90\", TimeNotSynchronized), (\"oxz_crucible_b202e971-de3b-4a73-a534-ee56c23e79a3\", TimeNotSynchronized), (\"oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e\", TimeNotSynchronized), (\"oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9\", TimeNotSynchronized), (\"oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189\", TimeNotSynchronized), (\"oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c\", TimeNotSynchronized), (\"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717\", TimeNotSynchronized), (\"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752\", TimeNotSynchronized), (\"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93\", TimeNotSynchronized), (\"oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02\", TimeNotSynchronized), (\"oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02\", TimeNotSynchronized), (\"oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4\", TimeNotSynchronized), (\"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3\", TimeNotSynchronized), (\"oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94\", TimeNotSynchronized), (\"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075\", TimeNotSynchronized), (\"oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139\", TimeNotSynchronized), (\"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f\", TimeNotSynchronized)]","latency_us":473043,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:27.568233163Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"3.773387803s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"8407801c-85d6-4354-82bf-e7d6eb7f2994\", \"content-length\": \"124\", \"date\": \"Wed, 26 Jun 2024 05:36:27 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"8407801c-85d6-4354-82bf-e7d6eb7f2994\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:27.901602097Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:423","component":"producer-server","component":"MetricsManager","sled_id":"8ead6dd9-9765-4776-9768-713d8773d37f","component":"SledAgent","file":"oximeter/producer/src/lib.rs:423","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"3.631001615s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 9f65db5e-d949-4342-9d90-6d691adea120, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: a4d358ae-9d04-40a4-b4a5-ac8418ace88d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 707beb4d-acae-4ff1-bb7e-97992526c5fb, kind: Service { id: a4d358ae-9d04-40a4-b4a5-ac8418ace88d }, name: Name(\"ntp-a4d358ae-9d04-40a4-b4a5-ac8418ace88d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 238, 27])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e3552cf6-27c7-4100-942b-c5877c673046, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } } } }, OmicronZoneConfig { id: 9a0125d1-b636-4039-9e54-dea22726a38a, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } } } }, OmicronZoneConfig { id: 7c851202-bc92-4a32-b946-68379ad242d7, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 653e8bac-3f5b-49fe-a9f9-e9f0546059e9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External } } } }, OmicronZoneConfig { id: e4ffe9aa-395e-4536-a4d0-5cf0bc814584, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 48ce280c-ebea-49f3-9829-dc9b4d018a02, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: c2791662-4c87-4a63-a67a-4d6489fb4eaf, kind: Service { id: 48ce280c-ebea-49f3-9829-dc9b4d018a02 }, name: Name(\"external-dns-48ce280c-ebea-49f3-9829-dc9b4d018a02\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 86])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 38cec1c6-d097-4604-b9cb-30a5fab57e35, kind: Service { id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94 }, name: Name(\"external-dns-9f29eb53-b3d5-4fc1-b4d8-a252757e0c94\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 215])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 67179f00-d639-4f5b-b3c6-71d676113139, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: a724cf8c-c466-4981-aa80-a17ab2349f83, kind: Service { id: 67179f00-d639-4f5b-b3c6-71d676113139 }, name: Name(\"nexus-67179f00-d639-4f5b-b3c6-71d676113139\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 55])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7e2a6156-04b4-4797-8bed-d0c5c9cbe752, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: e31530f0-72e9-4544-bfbb-847c405d46c8, kind: Service { id: 7e2a6156-04b4-4797-8bed-d0c5c9cbe752 }, name: Name(\"nexus-7e2a6156-04b4-4797-8bed-d0c5c9cbe752\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 20a0479c-19b7-40a0-a935-5ff5399f7d02, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 79c95d4d-cff5-4419-8f3e-a6e216736b73, kind: Service { id: 20a0479c-19b7-40a0-a935-5ff5399f7d02 }, name: Name(\"nexus-20a0479c-19b7-40a0-a935-5ff5399f7d02\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 249])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: e42e8525-00d2-488f-a732-5c9514a5c7b4, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 96bb9301-f50f-4259-8a1b-169ec98cbff3, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } } } }, OmicronZoneConfig { id: 966bee63-3642-4e7a-a159-17eb59b72717, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: da8be60e-c115-4289-8526-a1ad7b366b93, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: d72762b4-6a43-4e1c-92b1-b432c1731075, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: c3d43ade-9569-403a-a6cc-494ec947793f, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } } } }, OmicronZoneConfig { id: f09c8d14-475d-49d8-b03f-d3a0b7b880cc, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } } } }, OmicronZoneConfig { id: b202e971-de3b-4a73-a534-ee56c23e79a3, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2b78106a-b661-444a-a40f-011ae4f0911a, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External } } } }, OmicronZoneConfig { id: f64b0db4-4e9f-4666-b267-c9abe9f8b189, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 56d79ac6-bb86-49e7-905e-343e52dbbb90, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd06e234-5970-4365-a0e5-6cd29811daea (zpool), kind: External } } } }, OmicronZoneConfig { id: 010164b7-56ba-4dda-8f67-3d5e05751e2e, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3488ba32-8748-4793-9389-5ea882fa963c, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool), kind: External } } } }, OmicronZoneConfig { id: 89e132b1-d133-478e-a8f7-cf9f30b211c9, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:31.355948673Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:31.356231664Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:31.356239154Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9f65db5e-d949-4342-9d90-6d691adea120, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:31.379216701Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:31.379263399Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9f65db5e-d949-4342-9d90-6d691adea120, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:31.400224362Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:31.400270366Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e3552cf6-27c7-4100-942b-c5877c673046, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:31.419685451Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:31.419726655Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e3552cf6-27c7-4100-942b-c5877c673046, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a0125d1-b636-4039-9e54-dea22726a38a, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:31.438261473Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:31.438310046Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a0125d1-b636-4039-9e54-dea22726a38a, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c851202-bc92-4a32-b946-68379ad242d7, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:31.45613964Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:31.456237409Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7c851202-bc92-4a32-b946-68379ad242d7, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 653e8bac-3f5b-49fe-a9f9-e9f0546059e9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:31.47377411Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:31.473803436Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 653e8bac-3f5b-49fe-a9f9-e9f0546059e9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e4ffe9aa-395e-4536-a4d0-5cf0bc814584, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:31.490972676Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:31.491010571Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e4ffe9aa-395e-4536-a4d0-5cf0bc814584, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48ce280c-ebea-49f3-9829-dc9b4d018a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:31.510259203Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:31.51029811Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 48ce280c-ebea-49f3-9829-dc9b4d018a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:31.528676676Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:31.528722042Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 96bb9301-f50f-4259-8a1b-169ec98cbff3, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:31.547453942Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:31.547566669Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 96bb9301-f50f-4259-8a1b-169ec98cbff3, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c3d43ade-9569-403a-a6cc-494ec947793f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:31.567263752Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:31.567315775Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c3d43ade-9569-403a-a6cc-494ec947793f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f09c8d14-475d-49d8-b03f-d3a0b7b880cc, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:31.587156504Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:31.587194341Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f09c8d14-475d-49d8-b03f-d3a0b7b880cc, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b202e971-de3b-4a73-a534-ee56c23e79a3, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:31.606283886Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:31.606328287Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b202e971-de3b-4a73-a534-ee56c23e79a3, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b78106a-b661-444a-a40f-011ae4f0911a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:31.624113047Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:31.624148691Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2b78106a-b661-444a-a40f-011ae4f0911a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f64b0db4-4e9f-4666-b267-c9abe9f8b189, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:31.642198746Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:31.642233435Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f64b0db4-4e9f-4666-b267-c9abe9f8b189, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56d79ac6-bb86-49e7-905e-343e52dbbb90, dataset_name: DatasetName { pool_name: ZpoolName { id: bd06e234-5970-4365-a0e5-6cd29811daea (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:31.659760115Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:31.659801141Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 56d79ac6-bb86-49e7-905e-343e52dbbb90, dataset_name: DatasetName { pool_name: ZpoolName { id: bd06e234-5970-4365-a0e5-6cd29811daea (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 010164b7-56ba-4dda-8f67-3d5e05751e2e, dataset_name: DatasetName { pool_name: ZpoolName { id: e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:31.676755486Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:31.676792049Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 010164b7-56ba-4dda-8f67-3d5e05751e2e, dataset_name: DatasetName { pool_name: ZpoolName { id: e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3488ba32-8748-4793-9389-5ea882fa963c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:31.695413216Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:31.695474221Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3488ba32-8748-4793-9389-5ea882fa963c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89e132b1-d133-478e-a8f7-cf9f30b211c9, dataset_name: DatasetName { pool_name: ZpoolName { id: cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:31.714750842Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:31.714794155Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 89e132b1-d133-478e-a8f7-cf9f30b211c9, dataset_name: DatasetName { pool_name: ZpoolName { id: cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:31.734715465Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/968bbfe8-8122-4a85-96dd-954188a2f89d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:31.734753013Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/e8396e20-0493-4030-b55b-5c9e59a237cf/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:31.73496268Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:31.735102971Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:31.829180956Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:31.829265024Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:31.829274728Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:31.82928399Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:31.829303115Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:31.829312575Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:31.829335975Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:31.82935483Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:31.829364626Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:31.829376214Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_f09c8d14-475d-49d8-b03f-d3a0b7b880cc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:31.829384998Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_b202e971-de3b-4a73-a534-ee56c23e79a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:31.829398143Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:31.82940373Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:31.829411221Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:31.829424018Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:31.829432868Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:31.829440463Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:31.829512533Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:31.829519695Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:31.82954874Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"658ed430-6fe2-4941-9e9c-5b2cea1e0032","remote_addr":"[fd00:1122:3344:101::1]:39162","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4\", TimeNotSynchronized), (\"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752\", TimeNotSynchronized), (\"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075\", TimeNotSynchronized), (\"oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c\", TimeNotSynchronized), (\"oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90\", TimeNotSynchronized), (\"oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9\", TimeNotSynchronized), (\"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f\", TimeNotSynchronized), (\"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93\", TimeNotSynchronized), (\"oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139\", TimeNotSynchronized), (\"oxz_crucible_f09c8d14-475d-49d8-b03f-d3a0b7b880cc\", TimeNotSynchronized), (\"oxz_crucible_b202e971-de3b-4a73-a534-ee56c23e79a3\", TimeNotSynchronized), (\"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717\", TimeNotSynchronized), (\"oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a\", TimeNotSynchronized), (\"oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e\", TimeNotSynchronized), (\"oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94\", TimeNotSynchronized), (\"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3\", TimeNotSynchronized), (\"oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189\", TimeNotSynchronized), (\"oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02\", TimeNotSynchronized), (\"oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02\", TimeNotSynchronized)]","latency_us":471849,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:31.829759101Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"6.057552979s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"658ed430-6fe2-4941-9e9c-5b2cea1e0032\", \"content-length\": \"124\", \"date\": \"Wed, 26 Jun 2024 05:36:31 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"658ed430-6fe2-4941-9e9c-5b2cea1e0032\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 9f65db5e-d949-4342-9d90-6d691adea120, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: a4d358ae-9d04-40a4-b4a5-ac8418ace88d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 707beb4d-acae-4ff1-bb7e-97992526c5fb, kind: Service { id: a4d358ae-9d04-40a4-b4a5-ac8418ace88d }, name: Name(\"ntp-a4d358ae-9d04-40a4-b4a5-ac8418ace88d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 238, 27])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e3552cf6-27c7-4100-942b-c5877c673046, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } } } }, OmicronZoneConfig { id: 9a0125d1-b636-4039-9e54-dea22726a38a, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } } } }, OmicronZoneConfig { id: 7c851202-bc92-4a32-b946-68379ad242d7, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 653e8bac-3f5b-49fe-a9f9-e9f0546059e9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External } } } }, OmicronZoneConfig { id: e4ffe9aa-395e-4536-a4d0-5cf0bc814584, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 48ce280c-ebea-49f3-9829-dc9b4d018a02, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: c2791662-4c87-4a63-a67a-4d6489fb4eaf, kind: Service { id: 48ce280c-ebea-49f3-9829-dc9b4d018a02 }, name: Name(\"external-dns-48ce280c-ebea-49f3-9829-dc9b4d018a02\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 86])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 38cec1c6-d097-4604-b9cb-30a5fab57e35, kind: Service { id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94 }, name: Name(\"external-dns-9f29eb53-b3d5-4fc1-b4d8-a252757e0c94\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 215])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 67179f00-d639-4f5b-b3c6-71d676113139, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: a724cf8c-c466-4981-aa80-a17ab2349f83, kind: Service { id: 67179f00-d639-4f5b-b3c6-71d676113139 }, name: Name(\"nexus-67179f00-d639-4f5b-b3c6-71d676113139\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 55])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7e2a6156-04b4-4797-8bed-d0c5c9cbe752, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: e31530f0-72e9-4544-bfbb-847c405d46c8, kind: Service { id: 7e2a6156-04b4-4797-8bed-d0c5c9cbe752 }, name: Name(\"nexus-7e2a6156-04b4-4797-8bed-d0c5c9cbe752\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 20a0479c-19b7-40a0-a935-5ff5399f7d02, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 79c95d4d-cff5-4419-8f3e-a6e216736b73, kind: Service { id: 20a0479c-19b7-40a0-a935-5ff5399f7d02 }, name: Name(\"nexus-20a0479c-19b7-40a0-a935-5ff5399f7d02\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 249])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: e42e8525-00d2-488f-a732-5c9514a5c7b4, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 96bb9301-f50f-4259-8a1b-169ec98cbff3, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } } } }, OmicronZoneConfig { id: 966bee63-3642-4e7a-a159-17eb59b72717, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: da8be60e-c115-4289-8526-a1ad7b366b93, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: d72762b4-6a43-4e1c-92b1-b432c1731075, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: c3d43ade-9569-403a-a6cc-494ec947793f, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } } } }, OmicronZoneConfig { id: f09c8d14-475d-49d8-b03f-d3a0b7b880cc, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } } } }, OmicronZoneConfig { id: b202e971-de3b-4a73-a534-ee56c23e79a3, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2b78106a-b661-444a-a40f-011ae4f0911a, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External } } } }, OmicronZoneConfig { id: f64b0db4-4e9f-4666-b267-c9abe9f8b189, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 56d79ac6-bb86-49e7-905e-343e52dbbb90, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd06e234-5970-4365-a0e5-6cd29811daea (zpool), kind: External } } } }, OmicronZoneConfig { id: 010164b7-56ba-4dda-8f67-3d5e05751e2e, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3488ba32-8748-4793-9389-5ea882fa963c, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool), kind: External } } } }, OmicronZoneConfig { id: 89e132b1-d133-478e-a8f7-cf9f30b211c9, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:37.909253771Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:37.90983777Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:37.909853033Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9f65db5e-d949-4342-9d90-6d691adea120, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:37.932342437Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:37.932394723Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9f65db5e-d949-4342-9d90-6d691adea120, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:37.952994529Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:37.95306369Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e3552cf6-27c7-4100-942b-c5877c673046, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:37.972281735Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:37.972340545Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e3552cf6-27c7-4100-942b-c5877c673046, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a0125d1-b636-4039-9e54-dea22726a38a, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:37.991038739Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:37.991106409Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a0125d1-b636-4039-9e54-dea22726a38a, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c851202-bc92-4a32-b946-68379ad242d7, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:38.009971354Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:38.010028206Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7c851202-bc92-4a32-b946-68379ad242d7, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 653e8bac-3f5b-49fe-a9f9-e9f0546059e9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:38.029259532Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:38.029323396Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 653e8bac-3f5b-49fe-a9f9-e9f0546059e9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e4ffe9aa-395e-4536-a4d0-5cf0bc814584, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:38.047766863Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:38.04781283Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e4ffe9aa-395e-4536-a4d0-5cf0bc814584, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48ce280c-ebea-49f3-9829-dc9b4d018a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:38.06560947Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:38.065664409Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 48ce280c-ebea-49f3-9829-dc9b4d018a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:38.084691959Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:38.084743269Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 96bb9301-f50f-4259-8a1b-169ec98cbff3, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:38.103127178Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:38.103256374Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 96bb9301-f50f-4259-8a1b-169ec98cbff3, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c3d43ade-9569-403a-a6cc-494ec947793f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:38.122635176Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:38.122690022Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c3d43ade-9569-403a-a6cc-494ec947793f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f09c8d14-475d-49d8-b03f-d3a0b7b880cc, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:38.141242486Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:38.141362927Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f09c8d14-475d-49d8-b03f-d3a0b7b880cc, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b202e971-de3b-4a73-a534-ee56c23e79a3, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:38.160126905Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:38.160186972Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b202e971-de3b-4a73-a534-ee56c23e79a3, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b78106a-b661-444a-a40f-011ae4f0911a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:38.178674624Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:38.178726328Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2b78106a-b661-444a-a40f-011ae4f0911a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f64b0db4-4e9f-4666-b267-c9abe9f8b189, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:38.197133772Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:38.197193426Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f64b0db4-4e9f-4666-b267-c9abe9f8b189, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56d79ac6-bb86-49e7-905e-343e52dbbb90, dataset_name: DatasetName { pool_name: ZpoolName { id: bd06e234-5970-4365-a0e5-6cd29811daea (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:38.216057062Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:38.216110425Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 56d79ac6-bb86-49e7-905e-343e52dbbb90, dataset_name: DatasetName { pool_name: ZpoolName { id: bd06e234-5970-4365-a0e5-6cd29811daea (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 010164b7-56ba-4dda-8f67-3d5e05751e2e, dataset_name: DatasetName { pool_name: ZpoolName { id: e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:38.23416066Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:38.234218768Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 010164b7-56ba-4dda-8f67-3d5e05751e2e, dataset_name: DatasetName { pool_name: ZpoolName { id: e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3488ba32-8748-4793-9389-5ea882fa963c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:38.25288681Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:38.252946295Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3488ba32-8748-4793-9389-5ea882fa963c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89e132b1-d133-478e-a8f7-cf9f30b211c9, dataset_name: DatasetName { pool_name: ZpoolName { id: cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:38.271524385Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:38.271579979Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 89e132b1-d133-478e-a8f7-cf9f30b211c9, dataset_name: DatasetName { pool_name: ZpoolName { id: cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:38.29052606Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/968bbfe8-8122-4a85-96dd-954188a2f89d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:38.29058472Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/e8396e20-0493-4030-b55b-5c9e59a237cf/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:38.290882903Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:38.291024159Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:38.386498385Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:38.386546611Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:38.386554486Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:38.386560102Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:38.38657093Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:38.386574614Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_b202e971-de3b-4a73-a534-ee56c23e79a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:38.386591865Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:38.386597368Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:38.386601634Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:38.386606528Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:38.38661084Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:38.386615838Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:38.386620113Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:38.386625213Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:38.386629394Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:38.386633482Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:38.386637813Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:38.386646513Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:38.38665076Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_f09c8d14-475d-49d8-b03f-d3a0b7b880cc"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:38.386748272Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"98fecdcb-9eb8-4447-a1ac-8e1dc82d6b3d","remote_addr":"[fd00:1122:3344:101::1]:39162","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02\", TimeNotSynchronized), (\"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3\", TimeNotSynchronized), (\"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93\", TimeNotSynchronized), (\"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752\", TimeNotSynchronized), (\"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075\", TimeNotSynchronized), (\"oxz_crucible_b202e971-de3b-4a73-a534-ee56c23e79a3\", TimeNotSynchronized), (\"oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e\", TimeNotSynchronized), (\"oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189\", TimeNotSynchronized), (\"oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c\", TimeNotSynchronized), (\"oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9\", TimeNotSynchronized), (\"oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139\", TimeNotSynchronized), (\"oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02\", TimeNotSynchronized), (\"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717\", TimeNotSynchronized), (\"oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a\", TimeNotSynchronized), (\"oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90\", TimeNotSynchronized), (\"oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94\", TimeNotSynchronized), (\"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f\", TimeNotSynchronized), (\"oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4\", TimeNotSynchronized), (\"oxz_crucible_f09c8d14-475d-49d8-b03f-d3a0b7b880cc\", TimeNotSynchronized)]","latency_us":475622,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:38.386940866Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"2.940981237s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"98fecdcb-9eb8-4447-a1ac-8e1dc82d6b3d\", \"content-length\": \"124\", \"date\": \"Wed, 26 Jun 2024 05:36:37 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"98fecdcb-9eb8-4447-a1ac-8e1dc82d6b3d\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 9f65db5e-d949-4342-9d90-6d691adea120, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: a4d358ae-9d04-40a4-b4a5-ac8418ace88d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 707beb4d-acae-4ff1-bb7e-97992526c5fb, kind: Service { id: a4d358ae-9d04-40a4-b4a5-ac8418ace88d }, name: Name(\"ntp-a4d358ae-9d04-40a4-b4a5-ac8418ace88d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 238, 27])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e3552cf6-27c7-4100-942b-c5877c673046, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } } } }, OmicronZoneConfig { id: 9a0125d1-b636-4039-9e54-dea22726a38a, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } } } }, OmicronZoneConfig { id: 7c851202-bc92-4a32-b946-68379ad242d7, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 653e8bac-3f5b-49fe-a9f9-e9f0546059e9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External } } } }, OmicronZoneConfig { id: e4ffe9aa-395e-4536-a4d0-5cf0bc814584, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 48ce280c-ebea-49f3-9829-dc9b4d018a02, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: c2791662-4c87-4a63-a67a-4d6489fb4eaf, kind: Service { id: 48ce280c-ebea-49f3-9829-dc9b4d018a02 }, name: Name(\"external-dns-48ce280c-ebea-49f3-9829-dc9b4d018a02\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 86])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 38cec1c6-d097-4604-b9cb-30a5fab57e35, kind: Service { id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94 }, name: Name(\"external-dns-9f29eb53-b3d5-4fc1-b4d8-a252757e0c94\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 215])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 67179f00-d639-4f5b-b3c6-71d676113139, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: a724cf8c-c466-4981-aa80-a17ab2349f83, kind: Service { id: 67179f00-d639-4f5b-b3c6-71d676113139 }, name: Name(\"nexus-67179f00-d639-4f5b-b3c6-71d676113139\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 55])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7e2a6156-04b4-4797-8bed-d0c5c9cbe752, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: e31530f0-72e9-4544-bfbb-847c405d46c8, kind: Service { id: 7e2a6156-04b4-4797-8bed-d0c5c9cbe752 }, name: Name(\"nexus-7e2a6156-04b4-4797-8bed-d0c5c9cbe752\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 20a0479c-19b7-40a0-a935-5ff5399f7d02, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 79c95d4d-cff5-4419-8f3e-a6e216736b73, kind: Service { id: 20a0479c-19b7-40a0-a935-5ff5399f7d02 }, name: Name(\"nexus-20a0479c-19b7-40a0-a935-5ff5399f7d02\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 249])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: e42e8525-00d2-488f-a732-5c9514a5c7b4, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 96bb9301-f50f-4259-8a1b-169ec98cbff3, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } } } }, OmicronZoneConfig { id: 966bee63-3642-4e7a-a159-17eb59b72717, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: da8be60e-c115-4289-8526-a1ad7b366b93, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: d72762b4-6a43-4e1c-92b1-b432c1731075, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: c3d43ade-9569-403a-a6cc-494ec947793f, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } } } }, OmicronZoneConfig { id: f09c8d14-475d-49d8-b03f-d3a0b7b880cc, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } } } }, OmicronZoneConfig { id: b202e971-de3b-4a73-a534-ee56c23e79a3, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2b78106a-b661-444a-a40f-011ae4f0911a, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External } } } }, OmicronZoneConfig { id: f64b0db4-4e9f-4666-b267-c9abe9f8b189, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 56d79ac6-bb86-49e7-905e-343e52dbbb90, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd06e234-5970-4365-a0e5-6cd29811daea (zpool), kind: External } } } }, OmicronZoneConfig { id: 010164b7-56ba-4dda-8f67-3d5e05751e2e, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3488ba32-8748-4793-9389-5ea882fa963c, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool), kind: External } } } }, OmicronZoneConfig { id: 89e132b1-d133-478e-a8f7-cf9f30b211c9, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:41.338700423Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:41.339580737Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:41.339587777Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9f65db5e-d949-4342-9d90-6d691adea120, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:41.362010001Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:41.362053605Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9f65db5e-d949-4342-9d90-6d691adea120, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:41.381735924Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:41.381780643Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e3552cf6-27c7-4100-942b-c5877c673046, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:41.401335125Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:41.401377183Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e3552cf6-27c7-4100-942b-c5877c673046, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a0125d1-b636-4039-9e54-dea22726a38a, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:41.421077512Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:41.421121031Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a0125d1-b636-4039-9e54-dea22726a38a, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c851202-bc92-4a32-b946-68379ad242d7, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:41.44025752Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:41.440294571Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7c851202-bc92-4a32-b946-68379ad242d7, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 653e8bac-3f5b-49fe-a9f9-e9f0546059e9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:41.458981947Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:41.459023104Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 653e8bac-3f5b-49fe-a9f9-e9f0546059e9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e4ffe9aa-395e-4536-a4d0-5cf0bc814584, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:41.477430158Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:41.477471878Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e4ffe9aa-395e-4536-a4d0-5cf0bc814584, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48ce280c-ebea-49f3-9829-dc9b4d018a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:41.496407193Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:41.49643783Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 48ce280c-ebea-49f3-9829-dc9b4d018a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:41.514930004Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:41.515040403Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 96bb9301-f50f-4259-8a1b-169ec98cbff3, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:41.533984572Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:41.534016701Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 96bb9301-f50f-4259-8a1b-169ec98cbff3, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c3d43ade-9569-403a-a6cc-494ec947793f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:41.552718695Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:41.552764014Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c3d43ade-9569-403a-a6cc-494ec947793f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f09c8d14-475d-49d8-b03f-d3a0b7b880cc, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:41.571586832Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:41.571633783Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f09c8d14-475d-49d8-b03f-d3a0b7b880cc, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b202e971-de3b-4a73-a534-ee56c23e79a3, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:41.589786839Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:41.589830508Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b202e971-de3b-4a73-a534-ee56c23e79a3, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b78106a-b661-444a-a40f-011ae4f0911a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:41.609225078Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:41.609256841Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2b78106a-b661-444a-a40f-011ae4f0911a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f64b0db4-4e9f-4666-b267-c9abe9f8b189, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:41.629005452Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:41.629042963Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f64b0db4-4e9f-4666-b267-c9abe9f8b189, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56d79ac6-bb86-49e7-905e-343e52dbbb90, dataset_name: DatasetName { pool_name: ZpoolName { id: bd06e234-5970-4365-a0e5-6cd29811daea (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:41.647719675Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:41.647755179Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 56d79ac6-bb86-49e7-905e-343e52dbbb90, dataset_name: DatasetName { pool_name: ZpoolName { id: bd06e234-5970-4365-a0e5-6cd29811daea (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 010164b7-56ba-4dda-8f67-3d5e05751e2e, dataset_name: DatasetName { pool_name: ZpoolName { id: e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:41.665650384Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:41.665686131Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 010164b7-56ba-4dda-8f67-3d5e05751e2e, dataset_name: DatasetName { pool_name: ZpoolName { id: e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3488ba32-8748-4793-9389-5ea882fa963c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:41.685033057Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:41.685079408Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3488ba32-8748-4793-9389-5ea882fa963c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89e132b1-d133-478e-a8f7-cf9f30b211c9, dataset_name: DatasetName { pool_name: ZpoolName { id: cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:41.703302768Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:41.703332983Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 89e132b1-d133-478e-a8f7-cf9f30b211c9, dataset_name: DatasetName { pool_name: ZpoolName { id: cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:41.721986268Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/968bbfe8-8122-4a85-96dd-954188a2f89d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:41.722032038Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/e8396e20-0493-4030-b55b-5c9e59a237cf/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:41.722252637Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:41.722392346Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:41.813918198Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:41.81398779Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:41.813997034Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:41.814071113Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:41.814079251Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:41.814096061Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:41.814104536Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:41.814112589Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:41.814120539Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:41.814136515Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:41.814142993Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:41.814147925Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:41.814154065Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:41.81416029Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:41.814168184Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_f09c8d14-475d-49d8-b03f-d3a0b7b880cc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:41.814174279Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:41.814186579Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_b202e971-de3b-4a73-a534-ee56c23e79a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:41.814191369Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:41.81419766Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:41.814238067Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"dca8bbde-a8f2-4517-aef5-192fbec3384b","remote_addr":"[fd00:1122:3344:101::1]:39162","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4\", TimeNotSynchronized), (\"oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189\", TimeNotSynchronized), (\"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752\", TimeNotSynchronized), (\"oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e\", TimeNotSynchronized), (\"oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9\", TimeNotSynchronized), (\"oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02\", TimeNotSynchronized), (\"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f\", TimeNotSynchronized), (\"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717\", TimeNotSynchronized), (\"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075\", TimeNotSynchronized), (\"oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a\", TimeNotSynchronized), (\"oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90\", TimeNotSynchronized), (\"oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02\", TimeNotSynchronized), (\"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3\", TimeNotSynchronized), (\"oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c\", TimeNotSynchronized), (\"oxz_crucible_f09c8d14-475d-49d8-b03f-d3a0b7b880cc\", TimeNotSynchronized), (\"oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94\", TimeNotSynchronized), (\"oxz_crucible_b202e971-de3b-4a73-a534-ee56c23e79a3\", TimeNotSynchronized), (\"oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139\", TimeNotSynchronized), (\"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93\", TimeNotSynchronized)]","latency_us":473751,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:41.814434281Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"9.659632708s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"dca8bbde-a8f2-4517-aef5-192fbec3384b\", \"content-length\": \"124\", \"date\": \"Wed, 26 Jun 2024 05:36:41 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"dca8bbde-a8f2-4517-aef5-192fbec3384b\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:44.930066607Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:62368"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:44.930132787Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"a0777aef-684b-41ab-9792-5eabe367fdff","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:62368","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":42,"response_code":"200"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:46.657007999Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:423","component":"producer-server","component":"MetricsManager","sled_id":"8ead6dd9-9765-4776-9768-713d8773d37f","component":"SledAgent","file":"oximeter/producer/src/lib.rs:423","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"4.345333139s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 9f65db5e-d949-4342-9d90-6d691adea120, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: a4d358ae-9d04-40a4-b4a5-ac8418ace88d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 707beb4d-acae-4ff1-bb7e-97992526c5fb, kind: Service { id: a4d358ae-9d04-40a4-b4a5-ac8418ace88d }, name: Name(\"ntp-a4d358ae-9d04-40a4-b4a5-ac8418ace88d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 238, 27])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e3552cf6-27c7-4100-942b-c5877c673046, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } } } }, OmicronZoneConfig { id: 9a0125d1-b636-4039-9e54-dea22726a38a, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } } } }, OmicronZoneConfig { id: 7c851202-bc92-4a32-b946-68379ad242d7, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 653e8bac-3f5b-49fe-a9f9-e9f0546059e9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External } } } }, OmicronZoneConfig { id: e4ffe9aa-395e-4536-a4d0-5cf0bc814584, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 48ce280c-ebea-49f3-9829-dc9b4d018a02, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: c2791662-4c87-4a63-a67a-4d6489fb4eaf, kind: Service { id: 48ce280c-ebea-49f3-9829-dc9b4d018a02 }, name: Name(\"external-dns-48ce280c-ebea-49f3-9829-dc9b4d018a02\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 86])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 38cec1c6-d097-4604-b9cb-30a5fab57e35, kind: Service { id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94 }, name: Name(\"external-dns-9f29eb53-b3d5-4fc1-b4d8-a252757e0c94\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 215])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 67179f00-d639-4f5b-b3c6-71d676113139, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: a724cf8c-c466-4981-aa80-a17ab2349f83, kind: Service { id: 67179f00-d639-4f5b-b3c6-71d676113139 }, name: Name(\"nexus-67179f00-d639-4f5b-b3c6-71d676113139\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 55])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7e2a6156-04b4-4797-8bed-d0c5c9cbe752, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: e31530f0-72e9-4544-bfbb-847c405d46c8, kind: Service { id: 7e2a6156-04b4-4797-8bed-d0c5c9cbe752 }, name: Name(\"nexus-7e2a6156-04b4-4797-8bed-d0c5c9cbe752\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 20a0479c-19b7-40a0-a935-5ff5399f7d02, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 79c95d4d-cff5-4419-8f3e-a6e216736b73, kind: Service { id: 20a0479c-19b7-40a0-a935-5ff5399f7d02 }, name: Name(\"nexus-20a0479c-19b7-40a0-a935-5ff5399f7d02\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 249])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: e42e8525-00d2-488f-a732-5c9514a5c7b4, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 96bb9301-f50f-4259-8a1b-169ec98cbff3, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } } } }, OmicronZoneConfig { id: 966bee63-3642-4e7a-a159-17eb59b72717, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: da8be60e-c115-4289-8526-a1ad7b366b93, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: d72762b4-6a43-4e1c-92b1-b432c1731075, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: c3d43ade-9569-403a-a6cc-494ec947793f, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } } } }, OmicronZoneConfig { id: f09c8d14-475d-49d8-b03f-d3a0b7b880cc, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } } } }, OmicronZoneConfig { id: b202e971-de3b-4a73-a534-ee56c23e79a3, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2b78106a-b661-444a-a40f-011ae4f0911a, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External } } } }, OmicronZoneConfig { id: f64b0db4-4e9f-4666-b267-c9abe9f8b189, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 56d79ac6-bb86-49e7-905e-343e52dbbb90, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd06e234-5970-4365-a0e5-6cd29811daea (zpool), kind: External } } } }, OmicronZoneConfig { id: 010164b7-56ba-4dda-8f67-3d5e05751e2e, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3488ba32-8748-4793-9389-5ea882fa963c, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool), kind: External } } } }, OmicronZoneConfig { id: 89e132b1-d133-478e-a8f7-cf9f30b211c9, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:51.50885934Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:51.510206522Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:51.510214622Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9f65db5e-d949-4342-9d90-6d691adea120, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:51.530941024Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:51.530994612Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9f65db5e-d949-4342-9d90-6d691adea120, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:51.549605065Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:51.549662216Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e3552cf6-27c7-4100-942b-c5877c673046, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:51.569106855Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:51.569157294Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e3552cf6-27c7-4100-942b-c5877c673046, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a0125d1-b636-4039-9e54-dea22726a38a, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:51.588992425Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:51.589040249Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a0125d1-b636-4039-9e54-dea22726a38a, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c851202-bc92-4a32-b946-68379ad242d7, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:51.608199675Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:51.608255674Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7c851202-bc92-4a32-b946-68379ad242d7, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 653e8bac-3f5b-49fe-a9f9-e9f0546059e9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:51.626675295Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:51.626724712Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 653e8bac-3f5b-49fe-a9f9-e9f0546059e9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e4ffe9aa-395e-4536-a4d0-5cf0bc814584, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:51.644106096Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:51.644152757Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e4ffe9aa-395e-4536-a4d0-5cf0bc814584, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48ce280c-ebea-49f3-9829-dc9b4d018a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:51.662883582Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:51.662945355Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 48ce280c-ebea-49f3-9829-dc9b4d018a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:51.682521838Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:51.682568967Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 96bb9301-f50f-4259-8a1b-169ec98cbff3, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:51.702636788Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:51.702682295Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 96bb9301-f50f-4259-8a1b-169ec98cbff3, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c3d43ade-9569-403a-a6cc-494ec947793f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:51.721796963Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:51.721862796Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c3d43ade-9569-403a-a6cc-494ec947793f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f09c8d14-475d-49d8-b03f-d3a0b7b880cc, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:51.740243947Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:51.740291002Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f09c8d14-475d-49d8-b03f-d3a0b7b880cc, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b202e971-de3b-4a73-a534-ee56c23e79a3, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:51.758811736Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:51.758854393Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b202e971-de3b-4a73-a534-ee56c23e79a3, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b78106a-b661-444a-a40f-011ae4f0911a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:51.777584409Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:51.777645883Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2b78106a-b661-444a-a40f-011ae4f0911a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f64b0db4-4e9f-4666-b267-c9abe9f8b189, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:51.796650918Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:51.796704235Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f64b0db4-4e9f-4666-b267-c9abe9f8b189, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56d79ac6-bb86-49e7-905e-343e52dbbb90, dataset_name: DatasetName { pool_name: ZpoolName { id: bd06e234-5970-4365-a0e5-6cd29811daea (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:51.815482243Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:51.815544597Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 56d79ac6-bb86-49e7-905e-343e52dbbb90, dataset_name: DatasetName { pool_name: ZpoolName { id: bd06e234-5970-4365-a0e5-6cd29811daea (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 010164b7-56ba-4dda-8f67-3d5e05751e2e, dataset_name: DatasetName { pool_name: ZpoolName { id: e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:51.834400994Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:51.834459946Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 010164b7-56ba-4dda-8f67-3d5e05751e2e, dataset_name: DatasetName { pool_name: ZpoolName { id: e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3488ba32-8748-4793-9389-5ea882fa963c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:51.854121777Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:51.854170247Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3488ba32-8748-4793-9389-5ea882fa963c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89e132b1-d133-478e-a8f7-cf9f30b211c9, dataset_name: DatasetName { pool_name: ZpoolName { id: cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:51.872596433Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:51.872641773Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 89e132b1-d133-478e-a8f7-cf9f30b211c9, dataset_name: DatasetName { pool_name: ZpoolName { id: cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:51.890566343Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/968bbfe8-8122-4a85-96dd-954188a2f89d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:51.890621095Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/e8396e20-0493-4030-b55b-5c9e59a237cf/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:51.891031914Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:51.891179686Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:51.984611793Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:51.984655575Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:51.984666029Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:51.984670697Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:51.984674991Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_f09c8d14-475d-49d8-b03f-d3a0b7b880cc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:51.984683363Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:51.984688519Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:51.984692916Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:51.984697014Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_b202e971-de3b-4a73-a534-ee56c23e79a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:51.984705058Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:51.984709389Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:51.98471267Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:51.984720573Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:51.984725533Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:51.984730784Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:51.984742052Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:51.984747068Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:51.984752055Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:51.984756096Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:36:51.984837792Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"a8b24788-0aad-49ff-b8b4-0b2bc3099da1","remote_addr":"[fd00:1122:3344:101::1]:39162","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a\", TimeNotSynchronized), (\"oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189\", TimeNotSynchronized), (\"oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9\", TimeNotSynchronized), (\"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93\", TimeNotSynchronized), (\"oxz_crucible_f09c8d14-475d-49d8-b03f-d3a0b7b880cc\", TimeNotSynchronized), (\"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717\", TimeNotSynchronized), (\"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f\", TimeNotSynchronized), (\"oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4\", TimeNotSynchronized), (\"oxz_crucible_b202e971-de3b-4a73-a534-ee56c23e79a3\", TimeNotSynchronized), (\"oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90\", TimeNotSynchronized), (\"oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e\", TimeNotSynchronized), (\"oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c\", TimeNotSynchronized), (\"oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94\", TimeNotSynchronized), (\"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075\", TimeNotSynchronized), (\"oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139\", TimeNotSynchronized), (\"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752\", TimeNotSynchronized), (\"oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02\", TimeNotSynchronized), (\"oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02\", TimeNotSynchronized), (\"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3\", TimeNotSynchronized)]","latency_us":474175,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:36:51.98501007Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"8.533846445s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a8b24788-0aad-49ff-b8b4-0b2bc3099da1\", \"content-length\": \"124\", \"date\": \"Wed, 26 Jun 2024 05:36:51 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a8b24788-0aad-49ff-b8b4-0b2bc3099da1\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 9f65db5e-d949-4342-9d90-6d691adea120, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: a4d358ae-9d04-40a4-b4a5-ac8418ace88d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 707beb4d-acae-4ff1-bb7e-97992526c5fb, kind: Service { id: a4d358ae-9d04-40a4-b4a5-ac8418ace88d }, name: Name(\"ntp-a4d358ae-9d04-40a4-b4a5-ac8418ace88d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 238, 27])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e3552cf6-27c7-4100-942b-c5877c673046, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } } } }, OmicronZoneConfig { id: 9a0125d1-b636-4039-9e54-dea22726a38a, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } } } }, OmicronZoneConfig { id: 7c851202-bc92-4a32-b946-68379ad242d7, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 653e8bac-3f5b-49fe-a9f9-e9f0546059e9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External } } } }, OmicronZoneConfig { id: e4ffe9aa-395e-4536-a4d0-5cf0bc814584, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 48ce280c-ebea-49f3-9829-dc9b4d018a02, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: c2791662-4c87-4a63-a67a-4d6489fb4eaf, kind: Service { id: 48ce280c-ebea-49f3-9829-dc9b4d018a02 }, name: Name(\"external-dns-48ce280c-ebea-49f3-9829-dc9b4d018a02\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 86])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 38cec1c6-d097-4604-b9cb-30a5fab57e35, kind: Service { id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94 }, name: Name(\"external-dns-9f29eb53-b3d5-4fc1-b4d8-a252757e0c94\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 215])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 67179f00-d639-4f5b-b3c6-71d676113139, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: a724cf8c-c466-4981-aa80-a17ab2349f83, kind: Service { id: 67179f00-d639-4f5b-b3c6-71d676113139 }, name: Name(\"nexus-67179f00-d639-4f5b-b3c6-71d676113139\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 55])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7e2a6156-04b4-4797-8bed-d0c5c9cbe752, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: e31530f0-72e9-4544-bfbb-847c405d46c8, kind: Service { id: 7e2a6156-04b4-4797-8bed-d0c5c9cbe752 }, name: Name(\"nexus-7e2a6156-04b4-4797-8bed-d0c5c9cbe752\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 20a0479c-19b7-40a0-a935-5ff5399f7d02, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 79c95d4d-cff5-4419-8f3e-a6e216736b73, kind: Service { id: 20a0479c-19b7-40a0-a935-5ff5399f7d02 }, name: Name(\"nexus-20a0479c-19b7-40a0-a935-5ff5399f7d02\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 249])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: e42e8525-00d2-488f-a732-5c9514a5c7b4, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 96bb9301-f50f-4259-8a1b-169ec98cbff3, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } } } }, OmicronZoneConfig { id: 966bee63-3642-4e7a-a159-17eb59b72717, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: da8be60e-c115-4289-8526-a1ad7b366b93, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: d72762b4-6a43-4e1c-92b1-b432c1731075, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: c3d43ade-9569-403a-a6cc-494ec947793f, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } } } }, OmicronZoneConfig { id: f09c8d14-475d-49d8-b03f-d3a0b7b880cc, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } } } }, OmicronZoneConfig { id: b202e971-de3b-4a73-a534-ee56c23e79a3, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2b78106a-b661-444a-a40f-011ae4f0911a, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External } } } }, OmicronZoneConfig { id: f64b0db4-4e9f-4666-b267-c9abe9f8b189, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 56d79ac6-bb86-49e7-905e-343e52dbbb90, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd06e234-5970-4365-a0e5-6cd29811daea (zpool), kind: External } } } }, OmicronZoneConfig { id: 010164b7-56ba-4dda-8f67-3d5e05751e2e, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3488ba32-8748-4793-9389-5ea882fa963c, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool), kind: External } } } }, OmicronZoneConfig { id: 89e132b1-d133-478e-a8f7-cf9f30b211c9, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:00.549305112Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:00.550240688Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:00.550248609Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9f65db5e-d949-4342-9d90-6d691adea120, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:00.573045574Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:00.573088466Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9f65db5e-d949-4342-9d90-6d691adea120, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:00.592124528Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:00.592171967Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e3552cf6-27c7-4100-942b-c5877c673046, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:00.610891176Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:00.610926568Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e3552cf6-27c7-4100-942b-c5877c673046, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a0125d1-b636-4039-9e54-dea22726a38a, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:00.628537957Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:00.628575176Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a0125d1-b636-4039-9e54-dea22726a38a, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c851202-bc92-4a32-b946-68379ad242d7, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:00.645980193Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:00.646015818Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7c851202-bc92-4a32-b946-68379ad242d7, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 653e8bac-3f5b-49fe-a9f9-e9f0546059e9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:00.66330423Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:00.663338075Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 653e8bac-3f5b-49fe-a9f9-e9f0546059e9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e4ffe9aa-395e-4536-a4d0-5cf0bc814584, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:00.682224797Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:00.682257291Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e4ffe9aa-395e-4536-a4d0-5cf0bc814584, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48ce280c-ebea-49f3-9829-dc9b4d018a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:00.700625181Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:00.700657685Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 48ce280c-ebea-49f3-9829-dc9b4d018a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:00.718210559Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:00.718244141Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 96bb9301-f50f-4259-8a1b-169ec98cbff3, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:00.737036831Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:00.737068416Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 96bb9301-f50f-4259-8a1b-169ec98cbff3, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c3d43ade-9569-403a-a6cc-494ec947793f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:00.754920787Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:00.754971338Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c3d43ade-9569-403a-a6cc-494ec947793f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f09c8d14-475d-49d8-b03f-d3a0b7b880cc, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:00.773267127Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:00.77329784Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f09c8d14-475d-49d8-b03f-d3a0b7b880cc, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b202e971-de3b-4a73-a534-ee56c23e79a3, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:00.790568703Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:00.790603017Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b202e971-de3b-4a73-a534-ee56c23e79a3, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b78106a-b661-444a-a40f-011ae4f0911a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:00.809832678Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:00.809868248Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2b78106a-b661-444a-a40f-011ae4f0911a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f64b0db4-4e9f-4666-b267-c9abe9f8b189, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:00.827519341Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:00.827551994Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f64b0db4-4e9f-4666-b267-c9abe9f8b189, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56d79ac6-bb86-49e7-905e-343e52dbbb90, dataset_name: DatasetName { pool_name: ZpoolName { id: bd06e234-5970-4365-a0e5-6cd29811daea (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:00.845499596Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:00.845547615Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 56d79ac6-bb86-49e7-905e-343e52dbbb90, dataset_name: DatasetName { pool_name: ZpoolName { id: bd06e234-5970-4365-a0e5-6cd29811daea (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 010164b7-56ba-4dda-8f67-3d5e05751e2e, dataset_name: DatasetName { pool_name: ZpoolName { id: e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:00.863323047Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:00.863350423Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 010164b7-56ba-4dda-8f67-3d5e05751e2e, dataset_name: DatasetName { pool_name: ZpoolName { id: e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3488ba32-8748-4793-9389-5ea882fa963c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:00.881717835Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:00.881752327Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3488ba32-8748-4793-9389-5ea882fa963c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89e132b1-d133-478e-a8f7-cf9f30b211c9, dataset_name: DatasetName { pool_name: ZpoolName { id: cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:00.89968178Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:00.899719506Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 89e132b1-d133-478e-a8f7-cf9f30b211c9, dataset_name: DatasetName { pool_name: ZpoolName { id: cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:00.918202337Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/968bbfe8-8122-4a85-96dd-954188a2f89d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:00.918252456Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/e8396e20-0493-4030-b55b-5c9e59a237cf/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:00.918441592Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:00.918560723Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:01.00972104Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:01.009803401Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:01.009816827Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:01.009838062Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:01.00985671Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:01.009864556Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:01.009875338Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:01.009884328Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:01.009895054Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:01.009902892Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:01.009908348Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:01.009915679Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_f09c8d14-475d-49d8-b03f-d3a0b7b880cc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:01.009921407Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:01.009928954Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:01.009937815Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_b202e971-de3b-4a73-a534-ee56c23e79a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:01.009967984Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:01.009973618Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:01.009977565Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:01.009981672Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:01.009990428Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"b815f93a-1df0-4957-9f89-e2b52ec28fd0","remote_addr":"[fd00:1122:3344:101::1]:39162","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189\", TimeNotSynchronized), (\"oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90\", TimeNotSynchronized), (\"oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e\", TimeNotSynchronized), (\"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752\", TimeNotSynchronized), (\"oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c\", TimeNotSynchronized), (\"oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9\", TimeNotSynchronized), (\"oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02\", TimeNotSynchronized), (\"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717\", TimeNotSynchronized), (\"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93\", TimeNotSynchronized), (\"oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4\", TimeNotSynchronized), (\"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075\", TimeNotSynchronized), (\"oxz_crucible_f09c8d14-475d-49d8-b03f-d3a0b7b880cc\", TimeNotSynchronized), (\"oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02\", TimeNotSynchronized), (\"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f\", TimeNotSynchronized), (\"oxz_crucible_b202e971-de3b-4a73-a534-ee56c23e79a3\", TimeNotSynchronized), (\"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3\", TimeNotSynchronized), (\"oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94\", TimeNotSynchronized), (\"oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a\", TimeNotSynchronized), (\"oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139\", TimeNotSynchronized)]","latency_us":458967,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:01.010185892Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"5.851144849s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b815f93a-1df0-4957-9f89-e2b52ec28fd0\", \"content-length\": \"124\", \"date\": \"Wed, 26 Jun 2024 05:37:01 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b815f93a-1df0-4957-9f89-e2b52ec28fd0\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:06.104955262Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:423","component":"producer-server","component":"MetricsManager","sled_id":"8ead6dd9-9765-4776-9768-713d8773d37f","component":"SledAgent","file":"oximeter/producer/src/lib.rs:423","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"14.421099413s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 9f65db5e-d949-4342-9d90-6d691adea120, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: a4d358ae-9d04-40a4-b4a5-ac8418ace88d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 707beb4d-acae-4ff1-bb7e-97992526c5fb, kind: Service { id: a4d358ae-9d04-40a4-b4a5-ac8418ace88d }, name: Name(\"ntp-a4d358ae-9d04-40a4-b4a5-ac8418ace88d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 238, 27])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e3552cf6-27c7-4100-942b-c5877c673046, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } } } }, OmicronZoneConfig { id: 9a0125d1-b636-4039-9e54-dea22726a38a, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } } } }, OmicronZoneConfig { id: 7c851202-bc92-4a32-b946-68379ad242d7, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 653e8bac-3f5b-49fe-a9f9-e9f0546059e9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External } } } }, OmicronZoneConfig { id: e4ffe9aa-395e-4536-a4d0-5cf0bc814584, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 48ce280c-ebea-49f3-9829-dc9b4d018a02, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: c2791662-4c87-4a63-a67a-4d6489fb4eaf, kind: Service { id: 48ce280c-ebea-49f3-9829-dc9b4d018a02 }, name: Name(\"external-dns-48ce280c-ebea-49f3-9829-dc9b4d018a02\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 86])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 38cec1c6-d097-4604-b9cb-30a5fab57e35, kind: Service { id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94 }, name: Name(\"external-dns-9f29eb53-b3d5-4fc1-b4d8-a252757e0c94\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 215])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 67179f00-d639-4f5b-b3c6-71d676113139, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: a724cf8c-c466-4981-aa80-a17ab2349f83, kind: Service { id: 67179f00-d639-4f5b-b3c6-71d676113139 }, name: Name(\"nexus-67179f00-d639-4f5b-b3c6-71d676113139\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 55])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7e2a6156-04b4-4797-8bed-d0c5c9cbe752, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: e31530f0-72e9-4544-bfbb-847c405d46c8, kind: Service { id: 7e2a6156-04b4-4797-8bed-d0c5c9cbe752 }, name: Name(\"nexus-7e2a6156-04b4-4797-8bed-d0c5c9cbe752\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 20a0479c-19b7-40a0-a935-5ff5399f7d02, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 79c95d4d-cff5-4419-8f3e-a6e216736b73, kind: Service { id: 20a0479c-19b7-40a0-a935-5ff5399f7d02 }, name: Name(\"nexus-20a0479c-19b7-40a0-a935-5ff5399f7d02\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 249])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: e42e8525-00d2-488f-a732-5c9514a5c7b4, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 96bb9301-f50f-4259-8a1b-169ec98cbff3, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } } } }, OmicronZoneConfig { id: 966bee63-3642-4e7a-a159-17eb59b72717, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: da8be60e-c115-4289-8526-a1ad7b366b93, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: d72762b4-6a43-4e1c-92b1-b432c1731075, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: c3d43ade-9569-403a-a6cc-494ec947793f, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } } } }, OmicronZoneConfig { id: f09c8d14-475d-49d8-b03f-d3a0b7b880cc, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } } } }, OmicronZoneConfig { id: b202e971-de3b-4a73-a534-ee56c23e79a3, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2b78106a-b661-444a-a40f-011ae4f0911a, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External } } } }, OmicronZoneConfig { id: f64b0db4-4e9f-4666-b267-c9abe9f8b189, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 56d79ac6-bb86-49e7-905e-343e52dbbb90, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd06e234-5970-4365-a0e5-6cd29811daea (zpool), kind: External } } } }, OmicronZoneConfig { id: 010164b7-56ba-4dda-8f67-3d5e05751e2e, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3488ba32-8748-4793-9389-5ea882fa963c, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool), kind: External } } } }, OmicronZoneConfig { id: 89e132b1-d133-478e-a8f7-cf9f30b211c9, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:06.882115238Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:06.882377763Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:06.882385666Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9f65db5e-d949-4342-9d90-6d691adea120, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:06.904492569Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:06.907059984Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9f65db5e-d949-4342-9d90-6d691adea120, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:06.924861762Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:06.924988393Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e3552cf6-27c7-4100-942b-c5877c673046, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:06.943423334Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:06.94347501Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e3552cf6-27c7-4100-942b-c5877c673046, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a0125d1-b636-4039-9e54-dea22726a38a, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:06.961762869Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:06.961806708Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a0125d1-b636-4039-9e54-dea22726a38a, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c851202-bc92-4a32-b946-68379ad242d7, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:06.979500341Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:06.979550799Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7c851202-bc92-4a32-b946-68379ad242d7, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 653e8bac-3f5b-49fe-a9f9-e9f0546059e9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:06.998924577Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:06.998978222Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 653e8bac-3f5b-49fe-a9f9-e9f0546059e9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e4ffe9aa-395e-4536-a4d0-5cf0bc814584, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:07.018305373Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:07.018352436Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e4ffe9aa-395e-4536-a4d0-5cf0bc814584, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48ce280c-ebea-49f3-9829-dc9b4d018a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:07.037836751Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:07.037887602Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 48ce280c-ebea-49f3-9829-dc9b4d018a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:07.056325403Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:07.056435393Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 96bb9301-f50f-4259-8a1b-169ec98cbff3, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:07.072595243Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:07.072638828Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 96bb9301-f50f-4259-8a1b-169ec98cbff3, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c3d43ade-9569-403a-a6cc-494ec947793f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:07.091800551Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:07.091844981Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c3d43ade-9569-403a-a6cc-494ec947793f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f09c8d14-475d-49d8-b03f-d3a0b7b880cc, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:07.111854255Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:07.111899922Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f09c8d14-475d-49d8-b03f-d3a0b7b880cc, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b202e971-de3b-4a73-a534-ee56c23e79a3, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:07.130772007Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:07.130819436Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b202e971-de3b-4a73-a534-ee56c23e79a3, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b78106a-b661-444a-a40f-011ae4f0911a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:07.149006212Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:07.149057822Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2b78106a-b661-444a-a40f-011ae4f0911a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f64b0db4-4e9f-4666-b267-c9abe9f8b189, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:07.1665439Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:07.166595079Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f64b0db4-4e9f-4666-b267-c9abe9f8b189, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56d79ac6-bb86-49e7-905e-343e52dbbb90, dataset_name: DatasetName { pool_name: ZpoolName { id: bd06e234-5970-4365-a0e5-6cd29811daea (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:07.184562727Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:07.184608206Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 56d79ac6-bb86-49e7-905e-343e52dbbb90, dataset_name: DatasetName { pool_name: ZpoolName { id: bd06e234-5970-4365-a0e5-6cd29811daea (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 010164b7-56ba-4dda-8f67-3d5e05751e2e, dataset_name: DatasetName { pool_name: ZpoolName { id: e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:07.203004854Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:07.203061845Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 010164b7-56ba-4dda-8f67-3d5e05751e2e, dataset_name: DatasetName { pool_name: ZpoolName { id: e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3488ba32-8748-4793-9389-5ea882fa963c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:07.218657624Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:07.218704847Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3488ba32-8748-4793-9389-5ea882fa963c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89e132b1-d133-478e-a8f7-cf9f30b211c9, dataset_name: DatasetName { pool_name: ZpoolName { id: cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:07.238756202Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:07.238802103Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 89e132b1-d133-478e-a8f7-cf9f30b211c9, dataset_name: DatasetName { pool_name: ZpoolName { id: cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:07.256918651Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/968bbfe8-8122-4a85-96dd-954188a2f89d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:07.256977855Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/e8396e20-0493-4030-b55b-5c9e59a237cf/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:07.257205356Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:07.257326596Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:07.348051062Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:07.348103929Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:07.348109367Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:07.348114092Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:07.348118142Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:07.348121677Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:07.34812672Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:07.348134933Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:07.348139302Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:07.34814324Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:07.348152081Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_b202e971-de3b-4a73-a534-ee56c23e79a3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:07.348157171Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:07.348169893Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_f09c8d14-475d-49d8-b03f-d3a0b7b880cc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:07.348175059Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:07.348179296Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:07.348184256Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:07.348189197Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:07.348194128Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:07.348198075Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2960","zone":"oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:07.348276283Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"7d341d13-9757-4d35-baea-bfd6e2a921d6","remote_addr":"[fd00:1122:3344:101::1]:39162","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:866","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94\", TimeNotSynchronized), (\"oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139\", TimeNotSynchronized), (\"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3\", TimeNotSynchronized), (\"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93\", TimeNotSynchronized), (\"oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189\", TimeNotSynchronized), (\"oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02\", TimeNotSynchronized), (\"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717\", TimeNotSynchronized), (\"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075\", TimeNotSynchronized), (\"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f\", TimeNotSynchronized), (\"oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90\", TimeNotSynchronized), (\"oxz_crucible_b202e971-de3b-4a73-a534-ee56c23e79a3\", TimeNotSynchronized), (\"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752\", TimeNotSynchronized), (\"oxz_crucible_f09c8d14-475d-49d8-b03f-d3a0b7b880cc\", TimeNotSynchronized), (\"oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4\", TimeNotSynchronized), (\"oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c\", TimeNotSynchronized), (\"oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a\", TimeNotSynchronized), (\"oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9\", TimeNotSynchronized), (\"oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e\", TimeNotSynchronized), (\"oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02\", TimeNotSynchronized)]","latency_us":464469,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:07.348528263Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:469","retry_after":"16.553161143s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"7d341d13-9757-4d35-baea-bfd6e2a921d6\", \"content-length\": \"124\", \"date\": \"Wed, 26 Jun 2024 05:37:06 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"7d341d13-9757-4d35-baea-bfd6e2a921d6\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:15.061382739Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:63855"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:15.061455285Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"bf4954d3-42e1-4120-81b7-8f4b913788bb","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:63855","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":36,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, underlay_address: fd00:1122:3344:1::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 9f65db5e-d949-4342-9d90-6d691adea120, underlay_address: fd00:1122:3344:2::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, underlay_address: fd00:1122:3344:3::1, zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: a4d358ae-9d04-40a4-b4a5-ac8418ace88d, underlay_address: fd00:1122:3344:101::1b, zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 707beb4d-acae-4ff1-bb7e-97992526c5fb, kind: Service { id: a4d358ae-9d04-40a4-b4a5-ac8418ace88d }, name: Name(\"ntp-a4d358ae-9d04-40a4-b4a5-ac8418ace88d\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 238, 27])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: e3552cf6-27c7-4100-942b-c5877c673046, underlay_address: fd00:1122:3344:101::3, zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } } } }, OmicronZoneConfig { id: 9a0125d1-b636-4039-9e54-dea22726a38a, underlay_address: fd00:1122:3344:101::4, zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } } } }, OmicronZoneConfig { id: 7c851202-bc92-4a32-b946-68379ad242d7, underlay_address: fd00:1122:3344:101::5, zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 653e8bac-3f5b-49fe-a9f9-e9f0546059e9, underlay_address: fd00:1122:3344:101::6, zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External } } } }, OmicronZoneConfig { id: e4ffe9aa-395e-4536-a4d0-5cf0bc814584, underlay_address: fd00:1122:3344:101::7, zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 48ce280c-ebea-49f3-9829-dc9b4d018a02, underlay_address: fd00:1122:3344:101::8, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: c2791662-4c87-4a63-a67a-4d6489fb4eaf, kind: Service { id: 48ce280c-ebea-49f3-9829-dc9b4d018a02 }, name: Name(\"external-dns-48ce280c-ebea-49f3-9829-dc9b4d018a02\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 86])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94, underlay_address: fd00:1122:3344:101::9, zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 38cec1c6-d097-4604-b9cb-30a5fab57e35, kind: Service { id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94 }, name: Name(\"external-dns-9f29eb53-b3d5-4fc1-b4d8-a252757e0c94\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 215])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 } } }, OmicronZoneConfig { id: 67179f00-d639-4f5b-b3c6-71d676113139, underlay_address: fd00:1122:3344:101::a, zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: a724cf8c-c466-4981-aa80-a17ab2349f83, kind: Service { id: 67179f00-d639-4f5b-b3c6-71d676113139 }, name: Name(\"nexus-67179f00-d639-4f5b-b3c6-71d676113139\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 55])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 7e2a6156-04b4-4797-8bed-d0c5c9cbe752, underlay_address: fd00:1122:3344:101::b, zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: e31530f0-72e9-4544-bfbb-847c405d46c8, kind: Service { id: 7e2a6156-04b4-4797-8bed-d0c5c9cbe752 }, name: Name(\"nexus-7e2a6156-04b4-4797-8bed-d0c5c9cbe752\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 20a0479c-19b7-40a0-a935-5ff5399f7d02, underlay_address: fd00:1122:3344:101::c, zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 79c95d4d-cff5-4419-8f3e-a6e216736b73, kind: Service { id: 20a0479c-19b7-40a0-a935-5ff5399f7d02 }, name: Name(\"nexus-20a0479c-19b7-40a0-a935-5ff5399f7d02\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 249])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: e42e8525-00d2-488f-a732-5c9514a5c7b4, underlay_address: fd00:1122:3344:101::d, zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 96bb9301-f50f-4259-8a1b-169ec98cbff3, underlay_address: fd00:1122:3344:101::e, zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } } } }, OmicronZoneConfig { id: 966bee63-3642-4e7a-a159-17eb59b72717, underlay_address: fd00:1122:3344:101::f, zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: da8be60e-c115-4289-8526-a1ad7b366b93, underlay_address: fd00:1122:3344:101::10, zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: d72762b4-6a43-4e1c-92b1-b432c1731075, underlay_address: fd00:1122:3344:101::11, zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: c3d43ade-9569-403a-a6cc-494ec947793f, underlay_address: fd00:1122:3344:101::12, zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External } } } }, OmicronZoneConfig { id: f09c8d14-475d-49d8-b03f-d3a0b7b880cc, underlay_address: fd00:1122:3344:101::13, zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External } } } }, OmicronZoneConfig { id: b202e971-de3b-4a73-a534-ee56c23e79a3, underlay_address: fd00:1122:3344:101::14, zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External } } } }, OmicronZoneConfig { id: 2b78106a-b661-444a-a40f-011ae4f0911a, underlay_address: fd00:1122:3344:101::15, zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External } } } }, OmicronZoneConfig { id: f64b0db4-4e9f-4666-b267-c9abe9f8b189, underlay_address: fd00:1122:3344:101::16, zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External } } } }, OmicronZoneConfig { id: 56d79ac6-bb86-49e7-905e-343e52dbbb90, underlay_address: fd00:1122:3344:101::17, zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd06e234-5970-4365-a0e5-6cd29811daea (zpool), kind: External } } } }, OmicronZoneConfig { id: 010164b7-56ba-4dda-8f67-3d5e05751e2e, underlay_address: fd00:1122:3344:101::18, zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool), kind: External } } } }, OmicronZoneConfig { id: 3488ba32-8748-4793-9389-5ea882fa963c, underlay_address: fd00:1122:3344:101::19, zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool), kind: External } } } }, OmicronZoneConfig { id: 89e132b1-d133-478e-a8f7-cf9f30b211c9, underlay_address: fd00:1122:3344:101::1a, zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:23.951268728Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:431"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:23.951552713Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:23.951572945Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 09c6b6ed-da1f-491b-a5c1-f925ed53c317, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9f65db5e-d949-4342-9d90-6d691adea120, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:23.974963594Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:23.975091233Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9f65db5e-d949-4342-9d90-6d691adea120, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:23.993545213Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:23.993591049Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 217ccd12-3c92-4bac-9f9c-eac91bbdf1b2, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e3552cf6-27c7-4100-942b-c5877c673046, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:24.009556523Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:24.009621456Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e3552cf6-27c7-4100-942b-c5877c673046, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9a0125d1-b636-4039-9e54-dea22726a38a, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:24.030424748Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:24.030491715Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9a0125d1-b636-4039-9e54-dea22726a38a, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7c851202-bc92-4a32-b946-68379ad242d7, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:24.047050796Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:24.047115092Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 7c851202-bc92-4a32-b946-68379ad242d7, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 653e8bac-3f5b-49fe-a9f9-e9f0546059e9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:24.066521507Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:24.0665962Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 653e8bac-3f5b-49fe-a9f9-e9f0546059e9, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: e4ffe9aa-395e-4536-a4d0-5cf0bc814584, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:24.083089703Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:24.085015124Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: e4ffe9aa-395e-4536-a4d0-5cf0bc814584, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48ce280c-ebea-49f3-9829-dc9b4d018a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:24.102029437Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:24.102079501Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 48ce280c-ebea-49f3-9829-dc9b4d018a02, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:24.120986283Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:24.121034912Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 96bb9301-f50f-4259-8a1b-169ec98cbff3, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:24.139757756Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:24.139813361Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 96bb9301-f50f-4259-8a1b-169ec98cbff3, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c3d43ade-9569-403a-a6cc-494ec947793f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:24.159508304Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:24.159628513Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: c3d43ade-9569-403a-a6cc-494ec947793f, dataset_name: DatasetName { pool_name: ZpoolName { id: 2a12c7dc-fee7-4f85-ab9e-afd52f78b31f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f09c8d14-475d-49d8-b03f-d3a0b7b880cc, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:24.176660012Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:24.176715625Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f09c8d14-475d-49d8-b03f-d3a0b7b880cc, dataset_name: DatasetName { pool_name: ZpoolName { id: 5e9c007a-1be0-4b01-8d85-c9b89975656a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b202e971-de3b-4a73-a534-ee56c23e79a3, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:24.194227874Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:24.194271291Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: b202e971-de3b-4a73-a534-ee56c23e79a3, dataset_name: DatasetName { pool_name: ZpoolName { id: d4ac128c-bdc1-48a9-b5af-10262f74baf6 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2b78106a-b661-444a-a40f-011ae4f0911a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:24.21215722Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:24.212219388Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 2b78106a-b661-444a-a40f-011ae4f0911a, dataset_name: DatasetName { pool_name: ZpoolName { id: 4f34febf-41f3-4c60-ba39-4554d702f4af (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f64b0db4-4e9f-4666-b267-c9abe9f8b189, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:24.231641625Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:24.231703136Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: f64b0db4-4e9f-4666-b267-c9abe9f8b189, dataset_name: DatasetName { pool_name: ZpoolName { id: 9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 56d79ac6-bb86-49e7-905e-343e52dbbb90, dataset_name: DatasetName { pool_name: ZpoolName { id: bd06e234-5970-4365-a0e5-6cd29811daea (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:24.249921629Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:24.249975819Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 56d79ac6-bb86-49e7-905e-343e52dbbb90, dataset_name: DatasetName { pool_name: ZpoolName { id: bd06e234-5970-4365-a0e5-6cd29811daea (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 010164b7-56ba-4dda-8f67-3d5e05751e2e, dataset_name: DatasetName { pool_name: ZpoolName { id: e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:24.267644791Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:24.267705909Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 010164b7-56ba-4dda-8f67-3d5e05751e2e, dataset_name: DatasetName { pool_name: ZpoolName { id: e1e381d6-18e9-425e-b2e0-7789234ef986 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3488ba32-8748-4793-9389-5ea882fa963c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:24.286639477Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:24.286698982Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 3488ba32-8748-4793-9389-5ea882fa963c, dataset_name: DatasetName { pool_name: ZpoolName { id: 4031e1c2-96e5-4572-96c7-90799cc1b09a (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 89e132b1-d133-478e-a8f7-cf9f30b211c9, dataset_name: DatasetName { pool_name: ZpoolName { id: cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:24.305433149Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:24.305481957Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:809","request":"NewFilesystemRequest { dataset_id: 89e132b1-d133-478e-a8f7-cf9f30b211c9, dataset_name: DatasetName { pool_name: ZpoolName { id: cdd9c631-6644-4cda-aca6-05f7f580f308 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:24.32231939Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/968bbfe8-8122-4a85-96dd-954188a2f89d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:24.322391391Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/e8396e20-0493-4030-b55b-5c9e59a237cf/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:24.322616912Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:24.322748952Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:24.410692705Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:263"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:24.440116698Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:263"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:24.440424553Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:24.440467576Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:24.440493358Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:24.477737258Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:24.477782109Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:24.477786675Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:24.477790931Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:24.477795262Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:24.650339625Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:263"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:24.650561404Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:24.650603659Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:24.661032239Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:263"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:24.705674162Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:263"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:24.705941852Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:24.705979232Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:24.705993725Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:24.706015157Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:24.706349508Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:24.7101947Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:24.714351748Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:24.719457545Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:24.724572135Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:24.729397213Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:24.733957171Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:24.739600615Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:24.853573423Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:24.857960776Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:24.862355032Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:24.867233931Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:24.872149367Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:24.876933278Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:24.881217548Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:24.885765347Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Configuring new Omicron zone: oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:24.890763028Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:24.895142267Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:24.900260533Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_f09c8d14-475d-49d8-b03f-d3a0b7b880cc","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:24.905455156Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_b202e971-de3b-4a73-a534-ee56c23e79a3","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:24.910485075Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:24.916207803Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:24.925956387Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:24.932347182Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:24.93973564Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_f09c8d14-475d-49d8-b03f-d3a0b7b880cc","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:24.946046596Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_b202e971-de3b-4a73-a534-ee56c23e79a3","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:24.952011539Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:24.95875033Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:25.04664425Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:280","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:25.210505518Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:280","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:25.295245646Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:280","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:25.377651496Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:280","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:25.437305557Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:280","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:25.440782005Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:305"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:25.440798487Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:197","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:25.535423158Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:393","port":"Port { inner: PortInner { name: \"opte1\", ip: 172.30.2.7, mac: MacAddr6([168, 64, 37, 255, 246, 249]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte1\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:25.535540259Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:1190","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:25.54129486Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:305"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:25.541326023Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:197","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:25.633852107Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:393","port":"Port { inner: PortInner { name: \"opte2\", ip: 172.30.2.6, mac: MacAddr6([168, 64, 37, 255, 139, 118]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte2\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:25.633931262Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:1190","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:25.637615901Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:305"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:25.637642386Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:197","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:25.746964727Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:393","port":"Port { inner: PortInner { name: \"opte3\", ip: 172.30.1.6, mac: MacAddr6([168, 64, 37, 255, 186, 215]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 }, vnic: \"vopte3\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:25.747105321Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:1190","zone_type":"external_dns"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:25.749989037Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:305"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:25.750013093Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:197","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:25.927673974Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:393","port":"Port { inner: PortInner { name: \"opte4\", ip: 172.30.2.5, mac: MacAddr6([168, 64, 37, 255, 183, 55]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 }, vnic: \"vopte4\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:25.927740764Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:1190","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:25.930195208Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:305"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:25.930215195Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:197","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:26.17417591Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:393","port":"Port { inner: PortInner { name: \"opte5\", ip: 172.30.1.5, mac: MacAddr6([168, 64, 37, 255, 193, 86]), slot: 0, vni: Vni { inner: 100 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 }, vnic: \"vopte5\" } }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:26.174226565Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:1190","zone_type":"external_dns"} {"msg":"no nat entry found for: 10.151.1.168","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:26.174633711Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.168","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:26.174646715Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"no nat entry found for: 10.151.1.167","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:26.178428162Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.167","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:26.178443641Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"no nat entry found for: 10.151.1.165","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:26.178523358Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.165","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:26.178535012Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"no nat entry found for: 10.151.1.164","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:26.180720132Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.164","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:26.18073337Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"creation of nat entry successful for: 10.151.1.164","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:26.224736903Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:26.227356849Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"creation of nat entry successful for: 10.151.1.167","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:26.227384543Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"creation of nat entry successful for: 10.151.1.168","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:26.231553711Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"no nat entry found for: 10.151.1.166","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:26.235227555Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.166","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:26.235242668Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"creation of nat entry successful for: 10.151.1.165","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:26.235256233Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:26.235377286Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:26.235500687Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:26.334814115Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"creation of nat entry successful for: 10.151.1.166","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:26.444787766Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Configuring new Omicron zone: oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:26.478902578Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:26.484927219Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:26.485534083Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:26.493343608Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Configuring new Omicron zone: oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:26.529498013Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:26.535894958Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:26.542657638Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:26.547872269Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Configuring new Omicron zone: oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:26.553885983Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:26.560281377Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:26.586527528Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:28.645631119Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:28.645764772Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:28.64665097Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_b202e971-de3b-4a73-a534-ee56c23e79a3:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:28.646768187Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:28.648716717Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:28.656267572Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:28.662117154Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:28.667983184Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:28.668055139Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_b202e971-de3b-4a73-a534-ee56c23e79a3","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:28.674805687Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:28.675030157Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:28.675108271Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:28.681114335Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:28.687145987Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:28.773543778Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:28.7738018Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9\") failed: Property not found. retry in 60.532074ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:29.098213337Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90\") failed: Property not found. retry in 60.786826ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:29.11767289Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4\") failed: Property not found. retry in 69.419058ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:29.132522299Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c\") failed: Property not found. retry in 36.648324ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:29.147053974Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e\") failed: Property not found. retry in 69.76491ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:29.161113783Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a\") failed: Property not found. retry in 58.253428ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:29.175128416Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_crucible_f09c8d14-475d-49d8-b03f-d3a0b7b880cc:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:29.175449883Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_crucible_b202e971-de3b-4a73-a534-ee56c23e79a3\") failed: Property not found. retry in 52.681955ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:29.270497665Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_b202e971-de3b-4a73-a534-ee56c23e79a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189\") failed: Property not found. retry in 66.008589ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:31.245452015Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:31.249877841Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:31.250844103Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_f09c8d14-475d-49d8-b03f-d3a0b7b880cc","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_b202e971-de3b-4a73-a534-ee56c23e79a3\") failed: Property not found. retry in 101.018454ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:31.683111457Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_b202e971-de3b-4a73-a534-ee56c23e79a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189\") failed: Property not found. retry in 145.863415ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:31.72783484Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90\") failed: Property not found. retry in 105.972407ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:31.804738724Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9\") failed: Property not found. retry in 146.530237ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:31.84226997Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:31.844110763Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4\") failed: Property not found. retry in 100.233595ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:31.876870517Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c\") failed: Property not found. retry in 77.823959ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:31.920503487Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:31.920934953Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:31.922187334Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e\") failed: Property not found. retry in 110.190356ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:34.700500649Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a\") failed: Property not found. retry in 109.823671ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:34.718950469Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:34.720015448Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:34.720278854Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_f09c8d14-475d-49d8-b03f-d3a0b7b880cc\") failed: Property not found. retry in 60.586418ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:34.746894271Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_f09c8d14-475d-49d8-b03f-d3a0b7b880cc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b202e971-de3b-4a73-a534-ee56c23e79a3\") failed: Property not found. retry in 212.502729ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:34.759607488Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_b202e971-de3b-4a73-a534-ee56c23e79a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189\") failed: Property not found. retry in 119.342509ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:34.77129578Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90\") failed: Property not found. retry in 272.194732ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:34.783149817Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9\") failed: Property not found. retry in 287.71169ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:34.794528142Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:34.794584284Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4\") failed: Property not found. retry in 295.204108ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:34.8111429Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c\") failed: Property not found. retry in 142.516587ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:34.822796292Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:34.822861773Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:34.829359685Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e\") failed: Property not found. retry in 122.255321ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:34.845977172Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a\") failed: Property not found. retry in 231.904823ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:34.857397859Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:34.85746216Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_f09c8d14-475d-49d8-b03f-d3a0b7b880cc\") failed: Property not found. retry in 145.018414ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:34.874077448Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_f09c8d14-475d-49d8-b03f-d3a0b7b880cc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189\") failed: Property not found. retry in 298.051385ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:34.904195781Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93\") failed: Property not found. retry in 73.066248ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:34.948943609Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c\") failed: Property not found. retry in 344.695922ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:34.977702329Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e\") failed: Property not found. retry in 240.492903ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:34.989829418Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b202e971-de3b-4a73-a534-ee56c23e79a3\") failed: Property not found. retry in 541.474187ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:35.001654234Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_b202e971-de3b-4a73-a534-ee56c23e79a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075\") failed: Property not found. retry in 43.188886ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:35.012027526Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717\") failed: Property not found. retry in 46.881058ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:35.021998169Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f09c8d14-475d-49d8-b03f-d3a0b7b880cc\") failed: Property not found. retry in 265.668919ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:35.033410235Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_f09c8d14-475d-49d8-b03f-d3a0b7b880cc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93\") failed: Property not found. retry in 66.571409ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:35.043985192Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f\") failed: Property not found. retry in 28.12026ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:35.062452582Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90\") failed: Property not found. retry in 432.350511ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:35.082164412Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075\") failed: Property not found. retry in 142.826184ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:35.101701578Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9\") failed: Property not found. retry in 483.848159ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:35.117992488Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717\") failed: Property not found. retry in 61.8135ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:35.155716858Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f\") failed: Property not found. retry in 107.38833ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:35.17035072Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a\") failed: Property not found. retry in 471.112212ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:35.187709565Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94\") failed: Property not found. retry in 72.74755ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:35.200193656Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4\") failed: Property not found. retry in 297.410869ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:35.238702144Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717\") failed: Property not found. retry in 116.234306ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:35.25951219Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e\") failed: Property not found. retry in 590.134561ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:35.273124177Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93\") failed: Property not found. retry in 131.676042ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:35.286841478Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189\") failed: Property not found. retry in 1.162174275s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:35.297965277Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075\") failed: Property not found. retry in 265.027882ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:35.309852444Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f\") failed: Property not found. retry in 272.5252ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:35.320721928Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94\") failed: Property not found. retry in 128.667767ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:35.333992777Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f09c8d14-475d-49d8-b03f-d3a0b7b880cc\") failed: Property not found. retry in 288.804227ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:35.344897082Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_f09c8d14-475d-49d8-b03f-d3a0b7b880cc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:35.345146297Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c\") failed: Property not found. retry in 415.026493ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:35.354970187Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:35.355168191Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717\") failed: Property not found. retry in 452.3216ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:35.387076873Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93\") failed: Property not found. retry in 443.768608ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:35.431688246Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94\") failed: Property not found. retry in 284.36252ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:35.475030067Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90\") failed: Property not found. retry in 486.323545ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:35.538665832Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4\") failed: Property not found. retry in 678.630723ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:35.589977859Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:35.59089591Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_crucible_b202e971-de3b-4a73-a534-ee56c23e79a3\") failed: Property not found. retry in 558.747028ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:35.598276683Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_b202e971-de3b-4a73-a534-ee56c23e79a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:35.598339384Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f\") failed: Property not found. retry in 350.834693ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:35.618333781Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9\") failed: Property not found. retry in 715.119184ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:35.632220364Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075\") failed: Property not found. retry in 275.146028ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:35.646529743Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02\") failed: Property not found. retry in 56.593837ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:35.661789304Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f09c8d14-475d-49d8-b03f-d3a0b7b880cc\") failed: Property not found. retry in 673.620084ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:35.680835807Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_f09c8d14-475d-49d8-b03f-d3a0b7b880cc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a\") failed: Property not found. retry in 469.119151ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:35.691787954Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:35.703502889Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:423","component":"producer-server","component":"MetricsManager","sled_id":"8ead6dd9-9765-4776-9768-713d8773d37f","component":"SledAgent","file":"oximeter/producer/src/lib.rs:423","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"40.540595492s"} {"msg":"wait for service Some(\"oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02\") failed: Property not found. retry in 135.25115ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:35.730299402Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94\") failed: Property not found. retry in 526.168423ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:35.791265895Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c\") failed: Property not found. retry in 805.171111ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:35.813028512Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717\") failed: Property not found. retry in 414.303163ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:35.852454582Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e\") failed: Property not found. retry in 1.114370494s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:35.878135131Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02\") failed: Property not found. retry in 32.206851ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:35.890120726Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02\") failed: Property not found. retry in 221.518753ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:35.901469026Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93\") failed: Property not found. retry in 913.416091ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:35.910006455Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075\") failed: Property not found. retry in 553.58432ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:35.932198022Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02\") failed: Property not found. retry in 140.493049ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:35.940286218Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f\") failed: Property not found. retry in 591.37536ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:35.983608935Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90\") failed: Property not found. retry in 1.224615836s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:36.111766283Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02\") failed: Property not found. retry in 168.140956ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:36.126414777Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02\") failed: Property not found. retry in 576.594444ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:36.13741445Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b202e971-de3b-4a73-a534-ee56c23e79a3\") failed: Property not found. retry in 1.480933297s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:36.170662213Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_b202e971-de3b-4a73-a534-ee56c23e79a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a\") failed: Property not found. retry in 867.122466ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:36.182228065Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4\") failed: Property not found. retry in 985.335889ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:36.254780948Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717\") failed: Property not found. retry in 804.471823ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:36.28059679Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02\") failed: Property not found. retry in 203.403744ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:36.307780124Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94\") failed: Property not found. retry in 1.171800883s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:36.329758347Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9\") failed: Property not found. retry in 516.584957ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:36.362681705Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f09c8d14-475d-49d8-b03f-d3a0b7b880cc\") failed: Property not found. retry in 916.382879ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:36.375030843Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_f09c8d14-475d-49d8-b03f-d3a0b7b880cc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:36.386258255Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:36.386343407Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"Profile for oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:36.395662184Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:36.395731956Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189\") failed: Property not found. retry in 1.225354294s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:36.476150492Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075\") failed: Property not found. retry in 573.704129ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:36.503644223Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02\") failed: Property not found. retry in 650.52928ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:36.52750708Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f\") failed: Property not found. retry in 717.75439ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:36.599105241Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c\") failed: Property not found. retry in 921.117976ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:36.634484025Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752\") failed: Property not found. retry in 49.673619ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:36.646326171Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139\") failed: Property not found. retry in 37.711438ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:36.659193302Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752\") failed: Property not found. retry in 84.745272ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:36.709091506Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139\") failed: Property not found. retry in 115.485595ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:36.720531382Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02\") failed: Property not found. retry in 998.434548ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:36.737821846Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752\") failed: Property not found. retry in 252.895183ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:36.831966762Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93\") failed: Property not found. retry in 650.418817ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:36.887315508Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9\") failed: Property not found. retry in 953.736003ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:36.904506934Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139\") failed: Property not found. retry in 140.525894ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:36.918419639Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e\") failed: Property not found. retry in 587.322204ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:37.00675957Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a\") failed: Property not found. retry in 1.011928527s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:37.063666657Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139\") failed: Property not found. retry in 309.51496ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:37.075308859Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075\") failed: Property not found. retry in 1.33343337s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:37.091525437Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752\") failed: Property not found. retry in 406.745895ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:37.1029847Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717\") failed: Property not found. retry in 829.692546ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:37.115940137Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02\") failed: Property not found. retry in 891.980289ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:37.195861361Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4\") failed: Property not found. retry in 536.309067ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:37.260748287Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f09c8d14-475d-49d8-b03f-d3a0b7b880cc\") failed: Property not found. retry in 908.016749ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:37.307843334Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_f09c8d14-475d-49d8-b03f-d3a0b7b880cc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f\") failed: Property not found. retry in 1.181083136s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:37.332141149Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90\") failed: Property not found. retry in 1.111451338s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:37.354614187Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139\") failed: Property not found. retry in 720.01485ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:37.401336905Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94\") failed: Property not found. retry in 1.077734034s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:37.519615288Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752\") failed: Property not found. retry in 642.272599ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:37.539349715Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93\") failed: Property not found. retry in 636.010082ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:37.558932361Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c\") failed: Property not found. retry in 1.123315141s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:37.5747429Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e\") failed: Property not found. retry in 599.161999ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:37.614676992Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b202e971-de3b-4a73-a534-ee56c23e79a3\") failed: Property not found. retry in 1.109447801s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:37.674856465Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_b202e971-de3b-4a73-a534-ee56c23e79a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189\") failed: Property not found. retry in 1.112219267s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:37.71963049Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02\") failed: Property not found. retry in 1.422924964s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:37.756466366Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4\") failed: Property not found. retry in 1.365107079s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:37.81322259Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9\") failed: Property not found. retry in 1.15260649s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:37.874509163Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717\") failed: Property not found. retry in 736.760973ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:37.96453908Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a\") failed: Property not found. retry in 1.039184038s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:38.092160531Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02\") failed: Property not found. retry in 1.029595052s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:38.107657202Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139\") failed: Property not found. retry in 811.006247ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:38.137766604Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752\") failed: Property not found. retry in 558.822134ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:38.197094439Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93\") failed: Property not found. retry in 1.052724519s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:38.210513202Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e\") failed: Property not found. retry in 737.718944ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:38.228305807Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f09c8d14-475d-49d8-b03f-d3a0b7b880cc\") failed: Property not found. retry in 1.036827344s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:38.244180436Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_f09c8d14-475d-49d8-b03f-d3a0b7b880cc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075\") failed: Property not found. retry in 611.439738ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:38.440698512Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90\") failed: Property not found. retry in 853.463779ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:38.479257551Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f\") failed: Property not found. retry in 913.824651ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:38.52549967Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94\") failed: Property not found. retry in 797.475187ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:38.608615299Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c\") failed: Property not found. retry in 523.107682ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:38.710778619Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717\") failed: Property not found. retry in 1.01828205s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:38.723458358Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752\") failed: Property not found. retry in 887.244026ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:38.770978267Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b202e971-de3b-4a73-a534-ee56c23e79a3\") failed: Property not found. retry in 840.197002ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:38.799262267Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_b202e971-de3b-4a73-a534-ee56c23e79a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189\") failed: Property not found. retry in 829.85033ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:38.84269562Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139\") failed: Property not found. retry in 847.074288ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:38.965063594Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e\") failed: Property not found. retry in 508.384706ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:38.978782415Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9\") failed: Property not found. retry in 1.296970749s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:39.045293146Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075\") failed: Property not found. retry in 627.358737ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:39.065708552Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a\") failed: Property not found. retry in 531.657483ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:39.145563995Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02\") failed: Property not found. retry in 1.100743098s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:39.156252364Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4\") failed: Property not found. retry in 1.335682706s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:39.190861002Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02\") failed: Property not found. retry in 998.61912ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:39.203344213Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c\") failed: Property not found. retry in 715.254491ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:39.246196053Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93\") failed: Property not found. retry in 559.800057ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:39.276382227Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f09c8d14-475d-49d8-b03f-d3a0b7b880cc\") failed: Property not found. retry in 1.498794232s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:39.290569978Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_f09c8d14-475d-49d8-b03f-d3a0b7b880cc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90\") failed: Property not found. retry in 729.102025ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:39.346982634Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94\") failed: Property not found. retry in 1.102403078s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:39.421996875Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f\") failed: Property not found. retry in 655.763267ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:39.455590268Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e\") failed: Property not found. retry in 721.762325ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:39.500802964Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:39.591922648Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:39.592212125Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_crucible_b202e971-de3b-4a73-a534-ee56c23e79a3\") failed: Property not found. retry in 1.247106079s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:39.655528032Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_b202e971-de3b-4a73-a534-ee56c23e79a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752\") failed: Property not found. retry in 1.333796653s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:39.669423846Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189\") failed: Property not found. retry in 563.266763ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:39.686280359Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a\") failed: Property not found. retry in 1.169967461s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:39.701080135Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075\") failed: Property not found. retry in 942.645246ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:39.71673444Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717\") failed: Property not found. retry in 671.77102ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:39.757095572Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139\") failed: Property not found. retry in 649.35101ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:39.828527208Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93\") failed: Property not found. retry in 1.221127768s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:39.851194147Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3\") failed: Property not found. retry in 43.511439ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:39.860535885Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3\") failed: Property not found. retry in 65.430237ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:39.914020982Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c\") failed: Property not found. retry in 896.265036ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:39.973424269Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3\") failed: Property not found. retry in 188.927257ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:39.993238921Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90\") failed: Property not found. retry in 861.493269ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:40.095857484Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f\") failed: Property not found. retry in 792.794533ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:40.132366186Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3\") failed: Property not found. retry in 328.689411ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:40.194783832Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02\") failed: Property not found. retry in 546.734931ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:40.214710777Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e\") failed: Property not found. retry in 1.2326344s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:40.237942449Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189\") failed: Property not found. retry in 1.17711148s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:40.260838823Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02\") failed: Property not found. retry in 1.426561855s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:40.274699156Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9\") failed: Property not found. retry in 1.086630499s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:40.357214592Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717\") failed: Property not found. retry in 1.165887603s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:40.441193186Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139\") failed: Property not found. retry in 625.101292ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:40.489072878Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3\") failed: Property not found. retry in 1.040757319s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:40.535743926Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4\") failed: Property not found. retry in 1.035218242s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:40.547576773Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94\") failed: Property not found. retry in 1.335746646s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:40.560613157Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075\") failed: Property not found. retry in 649.893419ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:40.672924006Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02\") failed: Property not found. retry in 937.237937ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:40.773092182Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f09c8d14-475d-49d8-b03f-d3a0b7b880cc\") failed: Property not found. retry in 1.378244675s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:40.799821585Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_f09c8d14-475d-49d8-b03f-d3a0b7b880cc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c\") failed: Property not found. retry in 840.973809ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:40.880383296Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a\") failed: Property not found. retry in 1.440887501s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:40.88958632Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b202e971-de3b-4a73-a534-ee56c23e79a3\") failed: Property not found. retry in 715.610804ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:40.912389802Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_b202e971-de3b-4a73-a534-ee56c23e79a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f\") failed: Property not found. retry in 1.113035245s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:40.934859236Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90\") failed: Property not found. retry in 765.589685ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:40.96628922Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752\") failed: Property not found. retry in 832.300411ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:41.013526152Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93\") failed: Property not found. retry in 831.118712ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:41.081061077Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139\") failed: Property not found. retry in 1.204932534s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:41.122740636Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075\") failed: Property not found. retry in 809.555106ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:41.332351669Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189\") failed: Property not found. retry in 600.149681ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:41.448770455Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9\") failed: Property not found. retry in 546.076082ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:41.457525Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e\") failed: Property not found. retry in 1.312999489s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:41.479498692Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3\") failed: Property not found. retry in 511.385956ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:41.585732251Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4\") failed: Property not found. retry in 1.079180999s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:41.594321291Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717\") failed: Property not found. retry in 1.433186052s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:41.616905118Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b202e971-de3b-4a73-a534-ee56c23e79a3\") failed: Property not found. retry in 959.08612ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:41.637025667Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_b202e971-de3b-4a73-a534-ee56c23e79a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02\") failed: Property not found. retry in 1.361035836s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:41.711159786Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02\") failed: Property not found. retry in 1.181261463s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:41.719362039Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c\") failed: Property not found. retry in 813.666139ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:41.730121839Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90\") failed: Property not found. retry in 1.415659933s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:41.741513734Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752\") failed: Property not found. retry in 696.771986ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:41.855232011Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94\") failed: Property not found. retry in 904.418421ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:41.90675996Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93\") failed: Property not found. retry in 735.481861ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:41.920986041Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9\") failed: Property not found. retry in 802.716545ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:42.01337177Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f\") failed: Property not found. retry in 649.324093ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:42.059211266Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189\") failed: Property not found. retry in 577.799998ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:42.06754015Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3\") failed: Property not found. retry in 827.985861ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:42.106170337Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075\") failed: Property not found. retry in 580.972549ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:42.152161174Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f09c8d14-475d-49d8-b03f-d3a0b7b880cc\") failed: Property not found. retry in 1.043101398s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:42.187945629Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_f09c8d14-475d-49d8-b03f-d3a0b7b880cc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139\") failed: Property not found. retry in 1.317647852s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:42.338349928Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a\") failed: Property not found. retry in 598.453956ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:42.34658214Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c\") failed: Property not found. retry in 768.717138ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:42.554149392Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752\") failed: Property not found. retry in 1.473754293s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:42.562077717Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b202e971-de3b-4a73-a534-ee56c23e79a3\") failed: Property not found. retry in 1.028109208s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:42.604168309Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_b202e971-de3b-4a73-a534-ee56c23e79a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189\") failed: Property not found. retry in 778.220235ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:42.654617716Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93\") failed: Property not found. retry in 1.364931033s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:42.664970048Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4\") failed: Property not found. retry in 560.339694ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:42.681751709Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f\") failed: Property not found. retry in 929.203226ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:42.717548094Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075\") failed: Property not found. retry in 879.510529ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:42.743053359Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e\") failed: Property not found. retry in 1.319345158s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:42.802128174Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94\") failed: Property not found. retry in 1.027328428s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:42.820305221Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9\") failed: Property not found. retry in 581.413596ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:42.82882552Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02\") failed: Property not found. retry in 699.31407ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:42.91368376Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3\") failed: Property not found. retry in 1.333949286s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:42.947066129Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a\") failed: Property not found. retry in 593.714704ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:42.956580711Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717\") failed: Property not found. retry in 1.093558548s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:43.062521742Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02\") failed: Property not found. retry in 1.063259668s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:43.083914431Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90\") failed: Property not found. retry in 998.853212ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:43.169921837Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f09c8d14-475d-49d8-b03f-d3a0b7b880cc\") failed: Property not found. retry in 1.392006566s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:43.243107465Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_f09c8d14-475d-49d8-b03f-d3a0b7b880cc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4\") failed: Property not found. retry in 1.07807283s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:43.252485514Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c\") failed: Property not found. retry in 992.029544ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:43.33379936Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9\") failed: Property not found. retry in 967.037119ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:43.422592712Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189\") failed: Property not found. retry in 1.217794952s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:43.445742732Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a\") failed: Property not found. retry in 992.740765ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:43.562788753Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02\") failed: Property not found. retry in 917.052512ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:43.624443791Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075\") failed: Property not found. retry in 1.425950333s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:43.634310981Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b202e971-de3b-4a73-a534-ee56c23e79a3\") failed: Property not found. retry in 1.382575967s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:43.643750551Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_b202e971-de3b-4a73-a534-ee56c23e79a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f\") failed: Property not found. retry in 909.628682ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:43.657540433Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139\") failed: Property not found. retry in 786.748685ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:43.665784475Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94\") failed: Property not found. retry in 665.241356ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:43.857895797Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93\") failed: Property not found. retry in 1.306250331s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:44.040590159Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752\") failed: Property not found. retry in 578.683527ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:44.050040709Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e\") failed: Property not found. retry in 826.824643ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:44.130725034Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02\") failed: Property not found. retry in 532.216608ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:44.156468402Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717\") failed: Property not found. retry in 1.144315008s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:44.164913509Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90\") failed: Property not found. retry in 908.293691ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:44.176611192Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3\") failed: Property not found. retry in 573.154818ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:44.291115953Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c\") failed: Property not found. retry in 1.106238557s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:44.33597074Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4\") failed: Property not found. retry in 704.077737ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:44.343726671Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9\") failed: Property not found. retry in 1.490470708s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:44.399622385Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139\") failed: Property not found. retry in 1.479391454s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:44.464472448Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94\") failed: Property not found. retry in 1.193687753s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:44.533666312Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02\") failed: Property not found. retry in 1.371023476s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:44.551233132Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a\") failed: Property not found. retry in 1.430358878s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:44.564711356Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f\") failed: Property not found. retry in 1.416313103s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:44.576972752Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752\") failed: Property not found. retry in 775.849461ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:44.637395141Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f09c8d14-475d-49d8-b03f-d3a0b7b880cc\") failed: Property not found. retry in 1.129205169s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:44.646137445Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_f09c8d14-475d-49d8-b03f-d3a0b7b880cc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189\") failed: Property not found. retry in 660.840986ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:44.672765556Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02\") failed: Property not found. retry in 1.370566914s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:44.698194599Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3\") failed: Property not found. retry in 1.128826519s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:44.874992444Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e\") failed: Property not found. retry in 625.982361ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:44.966914518Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_b202e971-de3b-4a73-a534-ee56c23e79a3\") failed: Property not found. retry in 604.846296ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:45.042146529Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_b202e971-de3b-4a73-a534-ee56c23e79a3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4\") failed: Property not found. retry in 510.575568ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:45.059787765Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075\") failed: Property not found. retry in 1.103995835s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:45.071833186Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90\") failed: Property not found. retry in 784.433407ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:45.095079568Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:45.156989452Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:63574"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:45.285361537Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"8cfff04b-bb8c-4338-8a55-c12ba1fdd958","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:63574","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":128128,"response_code":"200"} {"msg":"wait for service Some(\"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717\") failed: Property not found. retry in 829.686205ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:45.320995689Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189\") failed: Property not found. retry in 1.329118628s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:45.343796896Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93\") failed: Property not found. retry in 1.099570217s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:45.357162739Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752\") failed: Property not found. retry in 1.141771079s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:45.424584285Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94\") failed: Property not found. retry in 1.104110844s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:45.839509555Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90\") failed: Property not found. retry in 1.450239124s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:46.009641374Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02\") failed: Property not found. retry in 906.816421ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:46.032113598Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3\") failed: Property not found. retry in 899.870678ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:46.041145449Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f\") failed: Property not found. retry in 1.355608221s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:46.049313099Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139\") failed: Property not found. retry in 1.290706903s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:46.071768303Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02\") failed: Property not found. retry in 1.100106038s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:46.081092221Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717\") failed: Property not found. retry in 628.30587ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:46.16291213Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075\") failed: Property not found. retry in 865.668132ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:46.186607952Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93\") failed: Property not found. retry in 1.017450654s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:46.466914482Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752\") failed: Property not found. retry in 1.214172559s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:46.576533779Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717\") failed: Property not found. retry in 848.389401ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:46.801776633Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02\") failed: Property not found. retry in 1.05790206s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:46.94910153Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3\") failed: Property not found. retry in 1.218101498s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:46.958264778Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94\") failed: Property not found. retry in 1.463309917s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:46.967185172Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075\") failed: Property not found. retry in 959.180423ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:47.063143631Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02\") failed: Property not found. retry in 1.099363912s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:47.190581893Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139\") failed: Property not found. retry in 1.153462573s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:47.373792967Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f\") failed: Property not found. retry in 979.734913ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:47.415561612Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93\") failed: Property not found. retry in 1.44487886s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:47.494503614Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717\") failed: Property not found. retry in 1.204728812s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:47.662058415Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752\") failed: Property not found. retry in 738.105904ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:47.802270313Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02\") failed: Property not found. retry in 854.207863ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:48.020783074Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3\") failed: Property not found. retry in 811.23115ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:48.188313479Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02\") failed: Property not found. retry in 997.402512ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:48.305983644Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f\") failed: Property not found. retry in 1.139957321s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:48.405824123Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139\") failed: Property not found. retry in 580.128225ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:48.538163517Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752\") failed: Property not found. retry in 953.92887ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:48.55061392Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3\") failed: Property not found. retry in 1.322594142s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:49.243317Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3\") failed: Property not found. retry in 1.195650404s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:50.577967316Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3\") failed: Property not found. retry in 552.577869ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:51.790965704Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3\") failed: Property not found. retry in 1.283666787s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:52.369114641Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3\") failed: Property not found. retry in 614.348781ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:37:53.665804161Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:54.309101208Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2956","zone":"oxz_nexus_7e2a6156-04b4-4797-8bed-d0c5c9cbe752"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:54.309168679Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2956","zone":"oxz_crucible_56d79ac6-bb86-49e7-905e-343e52dbbb90"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:54.30918887Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2956","zone":"oxz_crucible_89e132b1-d133-478e-a8f7-cf9f30b211c9"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:54.30919394Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2956","zone":"oxz_crucible_pantry_d72762b4-6a43-4e1c-92b1-b432c1731075"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:54.30919869Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2956","zone":"oxz_nexus_20a0479c-19b7-40a0-a935-5ff5399f7d02"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:54.30920301Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2956","zone":"oxz_oximeter_e42e8525-00d2-488f-a732-5c9514a5c7b4"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:54.30920755Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2956","zone":"oxz_crucible_3488ba32-8748-4793-9389-5ea882fa963c"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:54.30921165Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2956","zone":"oxz_clickhouse_96bb9301-f50f-4259-8a1b-169ec98cbff3"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:54.309215811Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2956","zone":"oxz_crucible_pantry_966bee63-3642-4e7a-a159-17eb59b72717"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:54.30922032Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2956","zone":"oxz_crucible_c3d43ade-9569-403a-a6cc-494ec947793f"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:54.309224551Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2956","zone":"oxz_crucible_010164b7-56ba-4dda-8f67-3d5e05751e2e"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:54.309229481Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2956","zone":"oxz_crucible_2b78106a-b661-444a-a40f-011ae4f0911a"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:54.309233601Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2956","zone":"oxz_nexus_67179f00-d639-4f5b-b3c6-71d676113139"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:54.309237701Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2956","zone":"oxz_external_dns_9f29eb53-b3d5-4fc1-b4d8-a252757e0c94"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:54.309242211Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2956","zone":"oxz_crucible_pantry_da8be60e-c115-4289-8526-a1ad7b366b93"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:54.309246441Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2956","zone":"oxz_crucible_f09c8d14-475d-49d8-b03f-d3a0b7b880cc"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:54.309250921Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2956","zone":"oxz_crucible_b202e971-de3b-4a73-a534-ee56c23e79a3"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:54.309255481Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2956","zone":"oxz_crucible_f64b0db4-4e9f-4666-b267-c9abe9f8b189"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:54.309263501Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2956","zone":"oxz_external_dns_48ce280c-ebea-49f3-9829-dc9b4d018a02"} {"msg":"Writing ledger to /pool/int/968bbfe8-8122-4a85-96dd-954188a2f89d/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:54.310078516Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/e8396e20-0493-4030-b55b-5c9e59a237cf/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:54.312784544Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:54.316009879Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"8c269a80-aac1-4fe0-aae6-004aa9fbc5b9","remote_addr":"[fd00:1122:3344:101::1]:39162","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":30341326,"response_code":"204"} {"msg":"Finished setting up services","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:54.329702408Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1244"} {"msg":"Writing ledger to /pool/int/968bbfe8-8122-4a85-96dd-954188a2f89d/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:54.330303082Z","hostname":"buskin","pid":1555,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/e8396e20-0493-4030-b55b-5c9e59a237cf/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:54.330524Z","hostname":"buskin","pid":1555,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Handing off control to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:54.336589259Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:681"} {"msg":"Nexus address: [fd00:1122:3344:101::c]:12221","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:54.341645397Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:694"} {"msg":"rack_network_config: RackNetworkConfigV2 {\n bfd: [],\n bgp: [],\n infra_ip_first: 10.151.1.174,\n infra_ip_last: 10.151.1.174,\n ports: [\n PortConfigV2 {\n addresses: [\n UplinkAddressConfig {\n address: V4(\n Ipv4Net {\n addr: 10.151.1.174,\n width: 24,\n },\n ),\n vlan_id: None,\n },\n ],\n autoneg: false,\n bgp_peers: [],\n port: \"qsfp0\",\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n nexthop: 10.151.1.100,\n vlan_id: None,\n },\n ],\n switch: Switch0,\n uplink_port_fec: None,\n uplink_port_speed: Speed40G,\n },\n ],\n rack_subnet: Ipv6Net {\n addr: fd00:1122:3344:100::,\n width: 56,\n },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:54.389290819Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:825"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/f0fff194-33a7-4fc7-ac24-138176f3dba5/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:54.398176506Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/f0fff194-33a7-4fc7-ac24-138176f3dba5/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:54.547464798Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/f0fff194-33a7-4fc7-ac24-138176f3dba5/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:54.713774258Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/f0fff194-33a7-4fc7-ac24-138176f3dba5/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:54.910241572Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/f0fff194-33a7-4fc7-ac24-138176f3dba5/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:55.060555598Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/f0fff194-33a7-4fc7-ac24-138176f3dba5/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:55.348925355Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/f0fff194-33a7-4fc7-ac24-138176f3dba5/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:55.485272546Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/f0fff194-33a7-4fc7-ac24-138176f3dba5/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:55.810658257Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/f0fff194-33a7-4fc7-ac24-138176f3dba5/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:56.238235112Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/f0fff194-33a7-4fc7-ac24-138176f3dba5/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:56.504602646Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/f0fff194-33a7-4fc7-ac24-138176f3dba5/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:57.24975813Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/f0fff194-33a7-4fc7-ac24-138176f3dba5/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:57.846319846Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/f0fff194-33a7-4fc7-ac24-138176f3dba5/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:58.867150992Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/8ead6dd9-9765-4776-9768-713d8773d37f): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-06-26T05:37:59.278783853Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"8ead6dd9-9765-4776-9768-713d8773d37f","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/f0fff194-33a7-4fc7-ac24-138176f3dba5/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:37:59.851539444Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/8ead6dd9-9765-4776-9768-713d8773d37f): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:38:00.257185667Z","hostname":"buskin","pid":1555,"component":"NexusNotifierTask","sled_id":"8ead6dd9-9765-4776-9768-713d8773d37f","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/8ead6dd9-9765-4776-9768-713d8773d37f): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-06-26T05:38:00.283564963Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"8ead6dd9-9765-4776-9768-713d8773d37f","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/f0fff194-33a7-4fc7-ac24-138176f3dba5/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:38:00.556213513Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/8ead6dd9-9765-4776-9768-713d8773d37f): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:38:00.701621285Z","hostname":"buskin","pid":1555,"component":"NexusNotifierTask","sled_id":"8ead6dd9-9765-4776-9768-713d8773d37f","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/8ead6dd9-9765-4776-9768-713d8773d37f): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-06-26T05:38:01.288760053Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"8ead6dd9-9765-4776-9768-713d8773d37f","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:169"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/f0fff194-33a7-4fc7-ac24-138176f3dba5/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:38:01.561359372Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a57c4e9c-2c1f-4fe3-a251-6b09f7b9adb8\", \"content-length\": \"173\", \"date\": \"Wed, 26 Jun 2024 05:38:03 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"8ead6dd9-9765-4776-9768-713d8773d37f\\\"\", request_id: \"a57c4e9c-2c1f-4fe3-a251-6b09f7b9adb8\" }","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:38:03.918457909Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:887"} {"msg":"Successfully put SledAgentInfo to nexus","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:38:04.733202606Z","hostname":"buskin","pid":1555,"component":"NexusNotifierTask","sled_id":"8ead6dd9-9765-4776-9768-713d8773d37f","component":"SledAgent","file":"sled-agent/src/nexus.rs:592","new_generation":"1","old_generation":"1"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:38:04.865935272Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:58261"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:38:04.867066759Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:38:04.954576391Z","hostname":"buskin","pid":1555,"uri":"/inventory","method":"GET","req_id":"b70f08cc-ae99-40c7-8e07-1a2bc2ddb632","remote_addr":"[fd00:1122:3344:101::c]:58261","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":88560,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:38:04.955095837Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/968bbfe8-8122-4a85-96dd-954188a2f89d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:38:04.955302541Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/e8396e20-0493-4030-b55b-5c9e59a237cf/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:38:04.957393606Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:38:04.957782905Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"GET","req_id":"24167ea2-36c0-4566-8797-57d40fff3837","remote_addr":"[fd00:1122:3344:101::c]:58261","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":2779,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:38:05.830023847Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:58542"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:38:05.835995393Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:524","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 38cec1c6-d097-4604-b9cb-30a5fab57e35, kind: Service { id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94 }, name: Name(\"external-dns-9f29eb53-b3d5-4fc1-b4d8-a252757e0c94\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 215])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: c2791662-4c87-4a63-a67a-4d6489fb4eaf, kind: Service { id: 48ce280c-ebea-49f3-9829-dc9b4d018a02 }, name: Name(\"external-dns-48ce280c-ebea-49f3-9829-dc9b4d018a02\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 86])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 79c95d4d-cff5-4419-8f3e-a6e216736b73, kind: Service { id: 20a0479c-19b7-40a0-a935-5ff5399f7d02 }, name: Name(\"nexus-20a0479c-19b7-40a0-a935-5ff5399f7d02\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 249])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: a724cf8c-c466-4981-aa80-a17ab2349f83, kind: Service { id: 67179f00-d639-4f5b-b3c6-71d676113139 }, name: Name(\"nexus-67179f00-d639-4f5b-b3c6-71d676113139\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 55])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: e31530f0-72e9-4544-bfbb-847c405d46c8, kind: Service { id: 7e2a6156-04b4-4797-8bed-d0c5c9cbe752 }, name: Name(\"nexus-7e2a6156-04b4-4797-8bed-d0c5c9cbe752\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:38:05.837176392Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:38:05.837311769Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:38:05.837347667Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:38:05.837354305Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:38:05.837366315Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:38:05.837376112Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:38:05.837391468Z","hostname":"buskin","pid":1555,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"7129af0a-99e9-4efb-a8ca-17ee36c5b9c2","remote_addr":"[fd00:1122:3344:101::c]:58542","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":7330,"response_code":"204"} {"msg":"Handoff to Nexus is complete","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:38:05.839956795Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:897"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:38:06.110829939Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:58553"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:38:06.111597632Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:38:06.202880432Z","hostname":"buskin","pid":1555,"uri":"/inventory","method":"GET","req_id":"bf23a084-b7ca-4449-a89e-5293fc910107","remote_addr":"[fd00:1122:3344:101::c]:58553","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":91883,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:38:06.203184007Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Reading ledger from /pool/int/968bbfe8-8122-4a85-96dd-954188a2f89d/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:38:06.203247927Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/e8396e20-0493-4030-b55b-5c9e59a237cf/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:38:06.20374126Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:38:06.204218563Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"GET","req_id":"b8a1b5dc-51e4-4340-ac46-86734a74dd75","remote_addr":"[fd00:1122:3344:101::c]:58553","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":1121,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:38:06.577429104Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:35513"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:38:06.577652113Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:524","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 38cec1c6-d097-4604-b9cb-30a5fab57e35, kind: Service { id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94 }, name: Name(\"external-dns-9f29eb53-b3d5-4fc1-b4d8-a252757e0c94\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 215])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: c2791662-4c87-4a63-a67a-4d6489fb4eaf, kind: Service { id: 48ce280c-ebea-49f3-9829-dc9b4d018a02 }, name: Name(\"external-dns-48ce280c-ebea-49f3-9829-dc9b4d018a02\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 86])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 79c95d4d-cff5-4419-8f3e-a6e216736b73, kind: Service { id: 20a0479c-19b7-40a0-a935-5ff5399f7d02 }, name: Name(\"nexus-20a0479c-19b7-40a0-a935-5ff5399f7d02\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 249])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: a724cf8c-c466-4981-aa80-a17ab2349f83, kind: Service { id: 67179f00-d639-4f5b-b3c6-71d676113139 }, name: Name(\"nexus-67179f00-d639-4f5b-b3c6-71d676113139\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 55])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: e31530f0-72e9-4544-bfbb-847c405d46c8, kind: Service { id: 7e2a6156-04b4-4797-8bed-d0c5c9cbe752 }, name: Name(\"nexus-7e2a6156-04b4-4797-8bed-d0c5c9cbe752\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:38:06.577705561Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:38:06.577735496Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:38:06.577759974Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:38:06.577766266Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:38:06.577771309Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:38:06.577783028Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:38:06.577797729Z","hostname":"buskin","pid":1555,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"1122c103-9489-4381-8d31-4bdf7681b047","remote_addr":"[fd00:1122:3344:101::c]:35513","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":351,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:38:06.71249451Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:60474"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:38:06.712858089Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:524","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 38cec1c6-d097-4604-b9cb-30a5fab57e35, kind: Service { id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94 }, name: Name(\"external-dns-9f29eb53-b3d5-4fc1-b4d8-a252757e0c94\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 215])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: c2791662-4c87-4a63-a67a-4d6489fb4eaf, kind: Service { id: 48ce280c-ebea-49f3-9829-dc9b4d018a02 }, name: Name(\"external-dns-48ce280c-ebea-49f3-9829-dc9b4d018a02\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 86])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 79c95d4d-cff5-4419-8f3e-a6e216736b73, kind: Service { id: 20a0479c-19b7-40a0-a935-5ff5399f7d02 }, name: Name(\"nexus-20a0479c-19b7-40a0-a935-5ff5399f7d02\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 249])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: a724cf8c-c466-4981-aa80-a17ab2349f83, kind: Service { id: 67179f00-d639-4f5b-b3c6-71d676113139 }, name: Name(\"nexus-67179f00-d639-4f5b-b3c6-71d676113139\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 55])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: e31530f0-72e9-4544-bfbb-847c405d46c8, kind: Service { id: 7e2a6156-04b4-4797-8bed-d0c5c9cbe752 }, name: Name(\"nexus-7e2a6156-04b4-4797-8bed-d0c5c9cbe752\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:38:06.71290931Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:38:06.712940391Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:38:06.712955431Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:38:06.712975201Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:38:06.712996002Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:38:06.713006432Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:38:06.713028568Z","hostname":"buskin","pid":1555,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"ea545e15-496a-405e-b977-169a656f261d","remote_addr":"[fd00:1122:3344:101::a]:60474","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":446,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:38:06.854596475Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:61608"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:38:06.855601849Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:524","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 38cec1c6-d097-4604-b9cb-30a5fab57e35, kind: Service { id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94 }, name: Name(\"external-dns-9f29eb53-b3d5-4fc1-b4d8-a252757e0c94\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 215])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: c2791662-4c87-4a63-a67a-4d6489fb4eaf, kind: Service { id: 48ce280c-ebea-49f3-9829-dc9b4d018a02 }, name: Name(\"external-dns-48ce280c-ebea-49f3-9829-dc9b4d018a02\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 86])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 79c95d4d-cff5-4419-8f3e-a6e216736b73, kind: Service { id: 20a0479c-19b7-40a0-a935-5ff5399f7d02 }, name: Name(\"nexus-20a0479c-19b7-40a0-a935-5ff5399f7d02\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 249])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: a724cf8c-c466-4981-aa80-a17ab2349f83, kind: Service { id: 67179f00-d639-4f5b-b3c6-71d676113139 }, name: Name(\"nexus-67179f00-d639-4f5b-b3c6-71d676113139\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 55])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: e31530f0-72e9-4544-bfbb-847c405d46c8, kind: Service { id: 7e2a6156-04b4-4797-8bed-d0c5c9cbe752 }, name: Name(\"nexus-7e2a6156-04b4-4797-8bed-d0c5c9cbe752\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:38:06.85566912Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:38:06.855699201Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:38:06.855705631Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:38:06.855711011Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:38:06.855721642Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:38:06.855728042Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:38:06.855750602Z","hostname":"buskin","pid":1555,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"7c5fdeb8-ebbf-48c9-a225-9caa9f13f898","remote_addr":"[fd00:1122:3344:101::b]:61608","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":1066,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:38:15.321060155Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:58574"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:38:15.321124517Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"53525d2c-4a7d-49e1-81a4-5aec4b90fc40","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:58574","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":48,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:38:32.381628468Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:56725"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:38:32.383651305Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"4eab94a4-8cfd-4388-8451-7bf6475e0c2e","remote_addr":"[fd00:1122:3344:101::c]:56725","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":2004,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:38:32.605530372Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:48040"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:38:32.719754947Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"61c31c81-e291-43e9-8b7e-e7be353df18c","remote_addr":"[fd00:1122:3344:101::c]:48040","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":114162,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:38:32.723976536Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"e168acb9-de65-418b-a0d0-1380604e7e17","remote_addr":"[fd00:1122:3344:101::c]:48040","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":3987,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:38:32.729355091Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:41988"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:38:32.729595547Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"11018d70-2cd3-4c0a-9316-81a9297a8f8a","remote_addr":"[fd00:1122:3344:101::b]:41988","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":178,"response_code":"200"} {"msg":"Attempting to update network config with generation: 2, current_generation: 1","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:38:32.744473564Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/968bbfe8-8122-4a85-96dd-954188a2f89d/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:38:32.744501815Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/e8396e20-0493-4030-b55b-5c9e59a237cf/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:38:32.746145683Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 2","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:38:32.747378652Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:38:32.747398532Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"PUT","req_id":"98506c61-ac58-4191-b488-37f2558507f0","remote_addr":"[fd00:1122:3344:101::c]:48040","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":3701,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:38:32.79359849Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:50110"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:38:32.793753714Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"b500d941-f5ce-4df7-ba3b-84c5a3c796bb","remote_addr":"[fd00:1122:3344:101::a]:50110","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":127,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:38:32.847805995Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:50863"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:38:32.896287316Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"5c5da985-18dd-45e3-8f49-146d21f61e1a","remote_addr":"[fd00:1122:3344:101::a]:50863","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":48442,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:38:32.896405779Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:62312"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:38:32.946210191Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"3f6f95b0-0f43-4e19-94c2-131c125f0a9f","remote_addr":"[fd00:1122:3344:101::b]:62312","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":49750,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:38:32.946401535Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"e9e673a6-eac9-489d-a499-3c187ba288ec","remote_addr":"[fd00:1122:3344:101::a]:50863","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":109,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:38:32.946638381Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"a33452ee-4f5a-40fc-b6db-3a7a78e55d6d","remote_addr":"[fd00:1122:3344:101::b]:62312","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":147,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:38:45.341875427Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:64244"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:38:45.342051331Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"ec68739f-608c-48b5-8517-042cdde92b41","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:64244","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":126,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:38:46.359988612Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:42379","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","component":"producer-server","component":"MetricsManager","sled_id":"8ead6dd9-9765-4776-9768-713d8773d37f","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::d]:64955"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:38:46.370305193Z","hostname":"buskin","pid":1555,"uri":"/8ead6dd9-9765-4776-9768-713d8773d37f","method":"GET","req_id":"b998ecb8-44e4-444d-8b6f-27391ec4412e","remote_addr":"[fd00:1122:3344:101::d]:64955","local_addr":"[fd00:1122:3344:101::1]:42379","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"8ead6dd9-9765-4776-9768-713d8773d37f","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":9712,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:39:02.491216388Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:55324"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:39:02.492385205Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"075b02c7-68da-4c01-841f-9aed35c184ac","remote_addr":"[fd00:1122:3344:101::c]:55324","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":1149,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:39:02.694475529Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:45515"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:39:02.745243084Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"a6ea639b-d5f7-4568-b92d-2042c6533bd1","remote_addr":"[fd00:1122:3344:101::c]:45515","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":50725,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:39:02.745606192Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:58441"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:39:02.745851238Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"31499433-d694-42ae-9606-3d69047dce16","remote_addr":"[fd00:1122:3344:101::c]:45515","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":271,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:39:02.745880759Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"155e77ac-9d86-4a1f-9707-689f227f0ba3","remote_addr":"[fd00:1122:3344:101::a]:58441","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":191,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:39:02.839637616Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:59871"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:39:02.839823751Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"bda5f8a0-f785-44f2-89f4-14ba672a93ef","remote_addr":"[fd00:1122:3344:101::b]:59871","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":172,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:39:03.062711171Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:42795"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:39:03.113073536Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"dea05587-cf95-4468-bb5d-a8b54f39995a","remote_addr":"[fd00:1122:3344:101::b]:42795","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":50305,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:39:03.113345672Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"298b93b0-2168-4469-adf2-283084e22ac3","remote_addr":"[fd00:1122:3344:101::b]:42795","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":124,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:39:03.189187252Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:37793"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:39:03.23752425Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"6a5b3169-af6f-4cff-b12b-a9c70cd7dbdb","remote_addr":"[fd00:1122:3344:101::a]:37793","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":48251,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:39:03.237794316Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"bebdd6f1-a148-4de2-8382-5aa9eb272b33","remote_addr":"[fd00:1122:3344:101::a]:37793","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":105,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:39:15.374831536Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:50251"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:39:15.374928769Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"14f5f7a9-3b48-4291-bc07-c3eb2a5a2f17","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:50251","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":92,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:39:16.358129359Z","hostname":"buskin","pid":1555,"uri":"/8ead6dd9-9765-4776-9768-713d8773d37f","method":"GET","req_id":"c0aed2fe-eda1-4cc5-9668-6ab38fa73c74","remote_addr":"[fd00:1122:3344:101::d]:64955","local_addr":"[fd00:1122:3344:101::1]:42379","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"8ead6dd9-9765-4776-9768-713d8773d37f","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":1841,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:39:32.339553576Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:38374"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:39:32.339790102Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"ef523583-d628-49e0-b9e0-3d111f17511f","remote_addr":"[fd00:1122:3344:101::c]:38374","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":196,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:39:32.4331323Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:35654"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:39:32.484312014Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"fcd8bfa3-ae64-47ad-a9b9-269a1881aeeb","remote_addr":"[fd00:1122:3344:101::c]:35654","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":51112,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:39:32.48458382Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"f2c83938-3b80-4cdd-b0ba-df19410b773b","remote_addr":"[fd00:1122:3344:101::c]:35654","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":94,"response_code":"200"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:39:32.654141236Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:39:32.654192937Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:47220"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:39:32.65516472Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"49440d2d-9119-4dfb-af51-a5f205aa99ed","remote_addr":"[fd00:1122:3344:101::a]:47220","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":913,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:39:32.711260959Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:64942"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:39:32.711379582Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"dba62c4e-1206-44ad-a8ad-1a042b0b2ab6","remote_addr":"[fd00:1122:3344:101::b]:64942","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":82,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:39:32.802956908Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:63072"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:39:32.851988383Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"5cb2fae7-f0f5-4109-9e86-f0d274e953ce","remote_addr":"[fd00:1122:3344:101::a]:63072","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":48965,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:39:32.852274889Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"1417fad7-e0ff-41d1-af89-cdd8ca6d572c","remote_addr":"[fd00:1122:3344:101::a]:63072","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":120,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:39:32.856080428Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:45812"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:39:32.901729433Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"a15e2653-4f6a-4603-b302-2814871e4faa","remote_addr":"[fd00:1122:3344:101::b]:45812","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":45563,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:39:32.901994659Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"32edb65f-8c5d-44a7-8cab-826d4828011d","remote_addr":"[fd00:1122:3344:101::b]:45812","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":94,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:39:45.413699174Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:49786"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:39:45.413792606Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"5398617e-8a3d-4184-9af0-a12ed1e838a1","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:49786","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":76,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:39:46.356409619Z","hostname":"buskin","pid":1555,"uri":"/8ead6dd9-9765-4776-9768-713d8773d37f","method":"GET","req_id":"1b53de4f-a655-4d74-95ab-4c041797c72b","remote_addr":"[fd00:1122:3344:101::d]:64955","local_addr":"[fd00:1122:3344:101::1]:42379","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"8ead6dd9-9765-4776-9768-713d8773d37f","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":595,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:40:02.451657824Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:34034"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:40:02.45191597Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"1b94380a-bf2c-4418-8259-88e6e4af4553","remote_addr":"[fd00:1122:3344:101::c]:34034","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":238,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:40:02.633882085Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:53790"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:40:02.689080312Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"587fbcaa-662b-4b2f-ab16-5d2845c9a41b","remote_addr":"[fd00:1122:3344:101::c]:53790","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":55132,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:40:02.689441051Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"0614701d-182d-4c54-a6d4-e1aeca1cd8e6","remote_addr":"[fd00:1122:3344:101::c]:53790","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":123,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:40:02.845554463Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:57132"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:40:02.845785839Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"da6a9c27-0dc3-400a-be5a-9cded62dcbb8","remote_addr":"[fd00:1122:3344:101::b]:57132","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":186,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:40:02.862442108Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:50365"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:40:02.862579091Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"0ef638b6-c556-4fae-bd43-55d8a644f03d","remote_addr":"[fd00:1122:3344:101::a]:50365","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":88,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:40:02.940677763Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:40194"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:40:02.987225649Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"06d863bf-1c18-4c59-ae93-af3fc63382c8","remote_addr":"[fd00:1122:3344:101::b]:40194","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":46495,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:40:02.987510006Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"d4d0f0ca-76bb-4be9-b6a4-95220093d84d","remote_addr":"[fd00:1122:3344:101::b]:40194","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":101,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:40:03.040630035Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:57600"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:40:03.087234962Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"48c694ea-f980-43a4-a4ca-8f7d25a41f4a","remote_addr":"[fd00:1122:3344:101::a]:57600","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":46488,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:40:03.087712894Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"58b38972-7b5c-4c13-bc1f-b535be8fe235","remote_addr":"[fd00:1122:3344:101::a]:57600","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":88,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:40:15.446361517Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:55907"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:40:15.446455939Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"2ddcf27a-2c8f-4411-8f59-54737d66ba32","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:55907","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":83,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:40:16.356575494Z","hostname":"buskin","pid":1555,"uri":"/8ead6dd9-9765-4776-9768-713d8773d37f","method":"GET","req_id":"480c3fde-eb16-431b-82fe-d7446a5ebf88","remote_addr":"[fd00:1122:3344:101::d]:64955","local_addr":"[fd00:1122:3344:101::1]:42379","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"8ead6dd9-9765-4776-9768-713d8773d37f","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":1050,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:40:32.38589002Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:36453"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:40:32.386067454Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"acfc3a29-791c-40af-be91-b098d24e3e59","remote_addr":"[fd00:1122:3344:101::c]:36453","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":163,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:40:32.464275869Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:36373"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:40:32.514172333Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"a8c9ea27-a9aa-4852-9632-4a5cc9eeb117","remote_addr":"[fd00:1122:3344:101::c]:36373","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":49831,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:40:32.51447917Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"4106d9f5-3c74-4498-b04e-e70e124bbe2c","remote_addr":"[fd00:1122:3344:101::c]:36373","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":105,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:40:32.612824755Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:48309"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:40:32.61304464Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"74f5f8ca-ee70-4f32-b357-ae8b93ee6ce5","remote_addr":"[fd00:1122:3344:101::a]:48309","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":179,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:40:32.70475538Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:56060"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:40:32.705370384Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"611a41b9-d267-4b18-b343-84024fade69d","remote_addr":"[fd00:1122:3344:101::b]:56060","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":624,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:40:32.778745646Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:41604"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:40:32.828173569Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"6c8d58e0-b7d5-4666-b936-ee7326ddf95e","remote_addr":"[fd00:1122:3344:101::b]:41604","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":49378,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:40:32.828297962Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:58555"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:40:32.832252164Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"c56c1589-6edf-4a91-ae20-c4fc7666730e","remote_addr":"[fd00:1122:3344:101::b]:41604","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":3742,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:40:32.877641803Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"ca32bf93-0f80-4f5d-bae3-89a0aac9b0ff","remote_addr":"[fd00:1122:3344:101::a]:58555","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":49255,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:40:32.877982451Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"4eaadb6e-ab8f-4bcc-be83-66795d4d2dbb","remote_addr":"[fd00:1122:3344:101::a]:58555","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":147,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:40:45.478961734Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:52895"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:40:45.480038049Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"f397832d-cb66-4dc6-984e-09e1e6e399d2","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:52895","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":1054,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:40:46.367960486Z","hostname":"buskin","pid":1555,"uri":"/8ead6dd9-9765-4776-9768-713d8773d37f","method":"GET","req_id":"d80ea5ca-bbf0-47ef-83b0-6956f4dd8dbc","remote_addr":"[fd00:1122:3344:101::d]:64955","local_addr":"[fd00:1122:3344:101::1]:42379","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"8ead6dd9-9765-4776-9768-713d8773d37f","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":317,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:41:02.403678025Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:64889"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:41:02.40388673Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"8d5548c4-d25e-4e1c-a6c4-69407da7b453","remote_addr":"[fd00:1122:3344:101::c]:64889","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":183,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:41:02.723396365Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:62363"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:41:02.723593289Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"db9edf7c-1896-49e2-bfa7-147b96718b6c","remote_addr":"[fd00:1122:3344:101::b]:62363","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":168,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:41:02.723666181Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:64759"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:41:02.777044956Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"ab5b0bd6-11df-423d-b97a-9c04a30dfc48","remote_addr":"[fd00:1122:3344:101::c]:64759","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":53298,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:41:02.777317003Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:47205"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:41:02.777593249Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"1bfcb261-747c-4e2e-ac2e-2bcdbc6b886a","remote_addr":"[fd00:1122:3344:101::c]:64759","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":221,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:41:02.777689351Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"65eece36-3b0a-4292-b006-be578c0c4b0c","remote_addr":"[fd00:1122:3344:101::a]:47205","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":267,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:41:02.974677548Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:47347"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:41:03.024349477Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"bfbd42e7-ecff-4f16-9fcd-1cbcf9698c73","remote_addr":"[fd00:1122:3344:101::b]:47347","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":49620,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:41:03.024595652Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"542e92b2-8188-4a36-90e5-0c71821a6f75","remote_addr":"[fd00:1122:3344:101::b]:47347","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":100,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:41:03.076428762Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:56061"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:41:03.124217447Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"43a9039c-4694-46b1-8673-3b0cd9ba4912","remote_addr":"[fd00:1122:3344:101::a]:56061","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":47712,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:41:03.124482193Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"b06c2d32-211a-42a5-bbd1-eab863e94f1c","remote_addr":"[fd00:1122:3344:101::a]:56061","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":100,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:41:15.511565368Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:64048"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:41:15.511635829Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"21deaf55-9907-4366-b9bb-af6457fa6707","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:64048","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":48,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:41:16.43014946Z","hostname":"buskin","pid":1555,"uri":"/8ead6dd9-9765-4776-9768-713d8773d37f","method":"GET","req_id":"1e108bb4-fff9-4fa4-b681-e7fd1260313d","remote_addr":"[fd00:1122:3344:101::d]:64955","local_addr":"[fd00:1122:3344:101::1]:42379","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"8ead6dd9-9765-4776-9768-713d8773d37f","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":857,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:41:32.366541294Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:41655"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:41:32.36676586Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"42783373-5026-4e70-a597-b1f2183bf77a","remote_addr":"[fd00:1122:3344:101::c]:41655","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":207,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:41:32.497324406Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:33300"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:41:32.550169889Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"109d9e4c-991b-4ddc-bbc3-012d8f373ed0","remote_addr":"[fd00:1122:3344:101::c]:33300","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":52777,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:41:32.550498167Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"67079f55-c1f9-45cb-98c8-4901d96181d7","remote_addr":"[fd00:1122:3344:101::c]:33300","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":101,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:41:32.636896042Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:56939"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:41:32.637117547Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"b4dcda1f-e712-42c3-98f7-9b3835a88af3","remote_addr":"[fd00:1122:3344:101::a]:56939","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":178,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:41:32.701678984Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:60695"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:41:32.750908852Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"fbe8ed3a-a966-4781-b790-4113c9eb2e36","remote_addr":"[fd00:1122:3344:101::a]:60695","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":49178,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:41:32.751014805Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:62475"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:41:32.75121745Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"bf53c5e6-7c1a-45e4-ab12-62f402280b7c","remote_addr":"[fd00:1122:3344:101::b]:62475","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":160,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:41:32.751897816Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"3f289ef3-ce53-434a-8bcd-aa9b4375faa8","remote_addr":"[fd00:1122:3344:101::a]:60695","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":749,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:41:32.802756502Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:51976"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:41:32.849225416Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"fb1a1f04-4617-4bc8-8df9-38f31c79f277","remote_addr":"[fd00:1122:3344:101::b]:51976","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":46393,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:41:32.849529274Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"55c652ae-d478-43ee-9582-21587981ca2d","remote_addr":"[fd00:1122:3344:101::b]:51976","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":105,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:41:45.549824944Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:50739"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:41:45.550711084Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"2a71d2c1-aaa5-44b6-a799-8a499813c6f7","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:50739","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":848,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:41:46.43087222Z","hostname":"buskin","pid":1555,"uri":"/8ead6dd9-9765-4776-9768-713d8773d37f","method":"GET","req_id":"5b51a1b8-92db-4019-b4d6-1a059fa49afc","remote_addr":"[fd00:1122:3344:101::d]:64955","local_addr":"[fd00:1122:3344:101::1]:42379","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"8ead6dd9-9765-4776-9768-713d8773d37f","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":1366,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:42:02.427854549Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:43454"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:42:02.428056753Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"a808bf16-8c9d-43aa-bfb6-4baec9da3bfc","remote_addr":"[fd00:1122:3344:101::c]:43454","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":176,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:42:02.668989695Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:60117"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:42:02.716689688Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"503a79de-34b7-42c4-a713-ea05818253ad","remote_addr":"[fd00:1122:3344:101::c]:60117","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":47644,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:42:02.716949614Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"e68b2ecb-1d37-4fd5-b5ea-4c588fd8f791","remote_addr":"[fd00:1122:3344:101::c]:60117","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":101,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:42:02.74208866Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:52481"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:42:02.742320456Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"70d79902-a508-4133-872c-b43e13deddd0","remote_addr":"[fd00:1122:3344:101::b]:52481","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":152,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:42:02.802728435Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:40631"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:42:02.80292046Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"b7b2a1eb-57be-4005-84df-ea0e6732b72a","remote_addr":"[fd00:1122:3344:101::a]:40631","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":137,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:42:03.009500249Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:44596"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:42:03.057652333Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"2d8a7476-ffd4-4d1e-b1b7-eec00311b48a","remote_addr":"[fd00:1122:3344:101::b]:44596","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":48076,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:42:03.057904129Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:57879"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:42:03.102312955Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"bcf54380-1489-438e-b221-5f8b0bd7e392","remote_addr":"[fd00:1122:3344:101::a]:57879","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":44334,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:42:03.102643023Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"2ecb67a9-8338-4a6e-84d1-a749810ab285","remote_addr":"[fd00:1122:3344:101::a]:57879","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":163,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:42:03.11107487Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"f947f9c3-ee80-45b0-981c-d7afbec5b99e","remote_addr":"[fd00:1122:3344:101::b]:44596","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":661,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:42:15.583004261Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:44741"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:42:15.583128904Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"c0eeb7af-4403-478d-89be-c121206d4671","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:44741","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":85,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:42:16.430607197Z","hostname":"buskin","pid":1555,"uri":"/8ead6dd9-9765-4776-9768-713d8773d37f","method":"GET","req_id":"9b2f2649-c6c1-4925-9c4f-42518d2bed1d","remote_addr":"[fd00:1122:3344:101::d]:64955","local_addr":"[fd00:1122:3344:101::1]:42379","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"8ead6dd9-9765-4776-9768-713d8773d37f","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":574,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:42:32.361390151Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:57112"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:42:32.362269041Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"231d14c7-7e28-477a-849a-8cedb10e1586","remote_addr":"[fd00:1122:3344:101::c]:57112","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":859,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:42:32.461473206Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:39219"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:42:32.511957484Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"b0f56745-a901-4fa0-8926-c2b0645891c0","remote_addr":"[fd00:1122:3344:101::c]:39219","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":50415,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:42:32.512341463Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"2ff7817f-81f7-4cfa-b3c5-c3ce968bea3e","remote_addr":"[fd00:1122:3344:101::c]:39219","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":127,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:42:32.609762435Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:60009"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:42:32.609984421Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"d24d31dd-933c-4789-bb99-35a8cb8374a0","remote_addr":"[fd00:1122:3344:101::a]:60009","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":181,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:42:32.728459825Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:40805"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:42:32.773167948Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"cde521f5-c2ae-4308-ab41-3ebeebf5bd19","remote_addr":"[fd00:1122:3344:101::a]:40805","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":44653,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:42:32.773283631Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:58306"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:42:32.774152451Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"4fc8747c-9a1f-4fa0-a0ee-66a177537411","remote_addr":"[fd00:1122:3344:101::b]:58306","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":816,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:42:32.774222643Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"e8a82ffe-b108-4e5f-87c8-4d6417141cde","remote_addr":"[fd00:1122:3344:101::a]:40805","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":786,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:42:32.893353362Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:47468"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:42:32.941727921Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"276ebf1f-0f28-48e8-ad08-8ac15526d7b5","remote_addr":"[fd00:1122:3344:101::b]:47468","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":48286,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:42:32.941988017Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"e0e908f4-c8a1-4494-a985-6029635a01cc","remote_addr":"[fd00:1122:3344:101::b]:47468","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":100,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:42:45.617085567Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:60280"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:42:45.617708621Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"11cbbcd0-6cf4-4cd4-b701-de6529a4ff49","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:60280","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":597,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:42:46.430035894Z","hostname":"buskin","pid":1555,"uri":"/8ead6dd9-9765-4776-9768-713d8773d37f","method":"GET","req_id":"27bde5f3-e7c0-4f34-9b2c-f1f8341112a8","remote_addr":"[fd00:1122:3344:101::d]:64955","local_addr":"[fd00:1122:3344:101::1]:42379","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"8ead6dd9-9765-4776-9768-713d8773d37f","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":606,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:43:02.650433555Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:59324"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:43:02.650677651Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"a63c7c52-66fb-4ee8-a820-9a17252e4b4a","remote_addr":"[fd00:1122:3344:101::c]:59324","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":207,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:43:02.713456465Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:43581"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:43:02.714605302Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:524","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 38cec1c6-d097-4604-b9cb-30a5fab57e35, kind: Service { id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94 }, name: Name(\"external-dns-9f29eb53-b3d5-4fc1-b4d8-a252757e0c94\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 215])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: c2791662-4c87-4a63-a67a-4d6489fb4eaf, kind: Service { id: 48ce280c-ebea-49f3-9829-dc9b4d018a02 }, name: Name(\"external-dns-48ce280c-ebea-49f3-9829-dc9b4d018a02\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 86])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 79c95d4d-cff5-4419-8f3e-a6e216736b73, kind: Service { id: 20a0479c-19b7-40a0-a935-5ff5399f7d02 }, name: Name(\"nexus-20a0479c-19b7-40a0-a935-5ff5399f7d02\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 249])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: a724cf8c-c466-4981-aa80-a17ab2349f83, kind: Service { id: 67179f00-d639-4f5b-b3c6-71d676113139 }, name: Name(\"nexus-67179f00-d639-4f5b-b3c6-71d676113139\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 55])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: e31530f0-72e9-4544-bfbb-847c405d46c8, kind: Service { id: 7e2a6156-04b4-4797-8bed-d0c5c9cbe752 }, name: Name(\"nexus-7e2a6156-04b4-4797-8bed-d0c5c9cbe752\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:43:02.714656693Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:43:02.714676954Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:43:02.714705894Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:43:02.714721085Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:43:02.714745215Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:43:02.714761306Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:43:02.714785766Z","hostname":"buskin","pid":1555,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"c012c499-a467-494c-814f-b0fd8767d2de","remote_addr":"[fd00:1122:3344:101::c]:43581","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":1290,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:43:02.753421858Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:41753"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:43:02.753578041Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"d0a0e538-ad91-4515-a92d-4a4e408c2f34","remote_addr":"[fd00:1122:3344:101::b]:41753","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":114,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:43:02.831661283Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:41016"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:43:02.883138524Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"63c0886b-3e28-4e2e-a668-5331bc9b7ee9","remote_addr":"[fd00:1122:3344:101::c]:41016","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":51393,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:43:02.883313378Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:54600"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:43:02.88340218Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:50676"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:43:02.883705597Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:524","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 38cec1c6-d097-4604-b9cb-30a5fab57e35, kind: Service { id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94 }, name: Name(\"external-dns-9f29eb53-b3d5-4fc1-b4d8-a252757e0c94\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 215])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: c2791662-4c87-4a63-a67a-4d6489fb4eaf, kind: Service { id: 48ce280c-ebea-49f3-9829-dc9b4d018a02 }, name: Name(\"external-dns-48ce280c-ebea-49f3-9829-dc9b4d018a02\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 86])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 79c95d4d-cff5-4419-8f3e-a6e216736b73, kind: Service { id: 20a0479c-19b7-40a0-a935-5ff5399f7d02 }, name: Name(\"nexus-20a0479c-19b7-40a0-a935-5ff5399f7d02\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 249])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: a724cf8c-c466-4981-aa80-a17ab2349f83, kind: Service { id: 67179f00-d639-4f5b-b3c6-71d676113139 }, name: Name(\"nexus-67179f00-d639-4f5b-b3c6-71d676113139\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 55])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: e31530f0-72e9-4544-bfbb-847c405d46c8, kind: Service { id: 7e2a6156-04b4-4797-8bed-d0c5c9cbe752 }, name: Name(\"nexus-7e2a6156-04b4-4797-8bed-d0c5c9cbe752\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:43:02.883771389Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:43:02.88382657Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:43:02.883866951Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:43:02.883904802Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:43:02.883936493Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:43:02.883975484Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:43:02.883994434Z","hostname":"buskin","pid":1555,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"7d685cde-4d56-47b6-90af-c475b4a9c5aa","remote_addr":"[fd00:1122:3344:101::a]:54600","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":594,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:43:02.884612869Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"ee347a4f-9b85-408f-8c4f-bf3d453433ee","remote_addr":"[fd00:1122:3344:101::c]:41016","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":1129,"response_code":"200"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:43:02.88465007Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:524","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 38cec1c6-d097-4604-b9cb-30a5fab57e35, kind: Service { id: 9f29eb53-b3d5-4fc1-b4d8-a252757e0c94 }, name: Name(\"external-dns-9f29eb53-b3d5-4fc1-b4d8-a252757e0c94\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 186, 215])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: c2791662-4c87-4a63-a67a-4d6489fb4eaf, kind: Service { id: 48ce280c-ebea-49f3-9829-dc9b4d018a02 }, name: Name(\"external-dns-48ce280c-ebea-49f3-9829-dc9b4d018a02\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 193, 86])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 79c95d4d-cff5-4419-8f3e-a6e216736b73, kind: Service { id: 20a0479c-19b7-40a0-a935-5ff5399f7d02 }, name: Name(\"nexus-20a0479c-19b7-40a0-a935-5ff5399f7d02\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 249])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: a724cf8c-c466-4981-aa80-a17ab2349f83, kind: Service { id: 67179f00-d639-4f5b-b3c6-71d676113139 }, name: Name(\"nexus-67179f00-d639-4f5b-b3c6-71d676113139\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 183, 55])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }, NetworkInterface { id: e31530f0-72e9-4544-bfbb-847c405d46c8, kind: Service { id: 7e2a6156-04b4-4797-8bed-d0c5c9cbe752 }, name: Name(\"nexus-7e2a6156-04b4-4797-8bed-d0c5c9cbe752\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 139, 118])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:43:02.884707561Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:43:02.884746172Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:43:02.884758882Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:43:02.884771002Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:43:02.884786523Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:43:02.884802633Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:543","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:43:02.884828474Z","hostname":"buskin","pid":1555,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"075f47d7-79ec-4201-92a7-55a0f78bd309","remote_addr":"[fd00:1122:3344:101::b]:50676","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":1211,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:43:02.906751565Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:60710"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:43:02.90695727Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"f03644fd-0979-41a0-8527-2091f10adba1","remote_addr":"[fd00:1122:3344:101::a]:60710","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":156,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:43:03.036908542Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:60599"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:43:03.087568624Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"a2eb4140-5fa6-4e81-99b7-d3e5b6d55423","remote_addr":"[fd00:1122:3344:101::b]:60599","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":50589,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:43:03.113708544Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"9ceae6c7-bd53-45eb-8d12-15bf9aac14b1","remote_addr":"[fd00:1122:3344:101::b]:60599","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":771,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:43:03.163914875Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:49382"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:43:03.208170838Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"b7f3ce85-21b7-47a1-8b28-92457325a6cf","remote_addr":"[fd00:1122:3344:101::a]:49382","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":44159,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:43:03.208441904Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"4bbcbc34-f7cf-4247-993d-bbac53499fe5","remote_addr":"[fd00:1122:3344:101::a]:49382","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":105,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:43:15.651146374Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:57697"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:43:15.651246016Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"4ea6876f-5085-4337-ab19-10c4be4b4232","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:57697","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":89,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:43:16.432442823Z","hostname":"buskin","pid":1555,"uri":"/8ead6dd9-9765-4776-9768-713d8773d37f","method":"GET","req_id":"971c0bcc-82ce-4ce3-a7a0-ac90ef74007b","remote_addr":"[fd00:1122:3344:101::d]:64955","local_addr":"[fd00:1122:3344:101::1]:42379","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"8ead6dd9-9765-4776-9768-713d8773d37f","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":2355,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:43:32.376392462Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:57661"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:43:32.376649718Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"c0dd60f2-df41-4692-b66f-d56ffe848496","remote_addr":"[fd00:1122:3344:101::c]:57661","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":234,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:43:32.510938301Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:50538"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:43:32.564521861Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"1feec522-88c4-48b5-a9f8-42d955820c33","remote_addr":"[fd00:1122:3344:101::c]:50538","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":53522,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:43:32.565777851Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"2825efde-d40d-416b-ba93-36adf57d1822","remote_addr":"[fd00:1122:3344:101::c]:50538","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":1019,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:43:32.740329483Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:55888"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:43:32.741175143Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"b98bba6d-fd90-4b9f-8fb1-108d050cad8b","remote_addr":"[fd00:1122:3344:101::b]:55888","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":801,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:43:32.743893706Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:40970"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:43:32.74402864Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"5d425003-1f66-4b0f-823a-a04eb3925538","remote_addr":"[fd00:1122:3344:101::a]:40970","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":95,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:43:32.806446486Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:34133"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:43:32.856732059Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"e30cd1a7-6a64-407e-aaea-aa9f0b704d6f","remote_addr":"[fd00:1122:3344:101::a]:34133","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":50209,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:43:32.856940174Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:50901"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:43:32.86018897Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"cceab193-7a81-4af2-8d37-e59742e28b1a","remote_addr":"[fd00:1122:3344:101::a]:34133","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":3096,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:43:32.902229931Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"288808fc-5f23-46ce-9855-2d25f589f280","remote_addr":"[fd00:1122:3344:101::b]:50901","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":45204,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:43:32.902584429Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"27827805-6ec9-4d1b-946b-3ce5105eef70","remote_addr":"[fd00:1122:3344:101::b]:50901","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":142,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:43:45.683351057Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:61072"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:43:45.683425309Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"19c94360-0576-4703-bb3e-6fbde3b999cd","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:61072","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":51,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:43:46.516686441Z","hostname":"buskin","pid":1555,"uri":"/8ead6dd9-9765-4776-9768-713d8773d37f","method":"GET","req_id":"060b746a-7b1a-4a55-8657-3eb16106cab1","remote_addr":"[fd00:1122:3344:101::d]:64955","local_addr":"[fd00:1122:3344:101::1]:42379","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"8ead6dd9-9765-4776-9768-713d8773d37f","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":897,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:02.41559992Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:42976"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:02.415810995Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"331778c6-e010-48ea-a9e5-a32ef5218a48","remote_addr":"[fd00:1122:3344:101::c]:42976","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":192,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:02.727210331Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:34433"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:02.727399595Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"8cd697d1-eca2-4427-b779-a7b8609214a1","remote_addr":"[fd00:1122:3344:101::b]:34433","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":168,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:02.740408119Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:55814"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:02.791405239Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"8f9f13d5-3d5d-40a5-9288-06392df7d99b","remote_addr":"[fd00:1122:3344:101::c]:55814","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":50932,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:02.791712176Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"d3ad6fe5-e3bf-41be-95b6-f6d80b5bcbac","remote_addr":"[fd00:1122:3344:101::c]:55814","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":125,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:02.839778737Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:52345"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:02.839987862Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"4283a961-b5fc-42f4-a7d3-7069ec9ab62b","remote_addr":"[fd00:1122:3344:101::a]:52345","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":169,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:02.95391122Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:33454"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:03.002056183Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"3da55828-bde5-45d2-8726-70449fe3e068","remote_addr":"[fd00:1122:3344:101::b]:33454","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":48084,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:03.002182846Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:34329"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:03.048489807Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"3f766921-e800-4c99-80c0-cdfb5f24d7af","remote_addr":"[fd00:1122:3344:101::a]:34329","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":46246,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:03.048702262Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"0543c487-438b-4eb8-baa5-1751609efaf2","remote_addr":"[fd00:1122:3344:101::b]:33454","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":111,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:03.048939617Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"f7119bf8-829c-4018-834e-fe07e44ddf1c","remote_addr":"[fd00:1122:3344:101::a]:34329","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":155,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:15.717711482Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:44071"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:15.717818155Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"214cb277-955b-4194-bff2-9e5ade5773b9","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:44071","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":93,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:16.515931437Z","hostname":"buskin","pid":1555,"uri":"/8ead6dd9-9765-4776-9768-713d8773d37f","method":"GET","req_id":"12d42aa7-8caf-4e7a-99ed-42635d36c63e","remote_addr":"[fd00:1122:3344:101::d]:64955","local_addr":"[fd00:1122:3344:101::1]:42379","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","component":"producer-server","component":"MetricsManager","sled_id":"8ead6dd9-9765-4776-9768-713d8773d37f","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":586,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:18.616353163Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:54003"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:18.616571078Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"6c8e53e7-4514-4017-8570-09aad852174d","remote_addr":"[fd00:1122:3344:101::a]:54003","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":201,"response_code":"200"} {"msg":"Mapping virtual NIC to physical host","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:18.618341559Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:631","mapping":"VirtualNetworkInterfaceHost { virtual_ip: 172.30.0.5, virtual_mac: MacAddr(MacAddr6([168, 64, 37, 254, 145, 89])), physical_host_ip: fd00:1122:3344:101::1, vni: Vni(5333119) }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:18.61838325Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"PUT","req_id":"ba31de23-26a4-45f7-aeea-5bc318672800","remote_addr":"[fd00:1122:3344:101::a]:54003","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":1645,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:18.760090286Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:37737"} {"msg":"ensuring instance is registered","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:18.772320702Z","hostname":"buskin","pid":1555,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:548","metadata":"InstanceMetadata { silo_id: 6334f5b4-1266-4926-aaf8-bf8279271639, project_id: 1efb3ce2-5910-46a8-8065-63018354c09d }","propolis_addr":"[fd00:1122:3344:101::1:0]:12400","vmm_runtime":"VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-06-26T05:44:18.359720Z }","instance_runtime":"InstanceRuntimeState { propolis_id: Some(6e15222a-e90a-4b46-9b34-bf4432694f7f (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-06-26T05:44:18.416175222Z }","hardware":"InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: 1815e9a0-21e4-4ca5-afdc-b1480b7be7b7, kind: Instance { id: c6a8d293-c1ab-4f47-8031-f4eef0d226b0 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 254, 145, 89])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(5333119), primary: true, slot: 0 }], source_nat: SourceNatConfig { ip: 10.151.1.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.1.176), floating_ips: [], firewall_rules: [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1815e9a0-21e4-4ca5-afdc-b1480b7be7b7, kind: Instance { id: c6a8d293-c1ab-4f47-8031-f4eef0d226b0 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 254, 145, 89])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(5333119), primary: true, slot: 0 }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1815e9a0-21e4-4ca5-afdc-b1480b7be7b7, kind: Instance { id: c6a8d293-c1ab-4f47-8031-f4eef0d226b0 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 254, 145, 89])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(5333119), primary: true, slot: 0 }], filter_hosts: Some([Vpc(Vni(5333119))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1815e9a0-21e4-4ca5-afdc-b1480b7be7b7, kind: Instance { id: c6a8d293-c1ab-4f47-8031-f4eef0d226b0 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 254, 145, 89])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(5333119), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-d195aefd226\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: d0e76d2b-2c2d-4fa5-b65f-d33bb26662b8, read_only_parent: Some(Volume { block_size: 512, id: e22a8cd6-5b7d-4e19-8268-372d04d938c1, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 670fe997-47c9-4248-9de8-4a9fdf55b094, key: Some(\"CtbdWQ4ppBxTkdQkeSSUojYrJWWDLDANyrbEi2bo5n0=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::13]:19001\", \"[fd00:1122:3344:101::16]:19002\", \"[fd00:1122:3344:101::1a]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: d0e76d2b-2c2d-4fa5-b65f-d33bb26662b8, key: Some(\"7C+xY8bGV1FC9OJKNGJAYZnviY0IGQ3JgwSlsPbxB6U=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::1a]:19002\", \"[fd00:1122:3344:101::16]:19003\", \"[fd00:1122:3344:101::12]:19001\"] } }] } }], cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAABIiS3aWNpYAACJLdpYAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAASIkt2ljaWAAAiS3aWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiJjNmE4ZDI5My1jMWFiLTRmNDctODAzMS1mNGVlZjBkMjI2YjAiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJSHErcSt5SldHRkp5NzlPaEU0TnY0RHBzWmMra1J1OEJrQWlrNllpY1didiJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\") }","propolis_id":"6e15222a-e90a-4b46-9b34-bf4432694f7f","instance_id":"c6a8d293-c1ab-4f47-8031-f4eef0d226b0"} {"msg":"registering new instance","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:18.772420054Z","hostname":"buskin","pid":1555,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:582","instance_id":"c6a8d293-c1ab-4f47-8031-f4eef0d226b0 (instance)"} {"msg":"initializing new Instance","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:18.773615962Z","hostname":"buskin","pid":1555,"component":"InstanceManager","file":"sled-agent/src/instance.rs:942","state":"InstanceInitialState { hardware: InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: 1815e9a0-21e4-4ca5-afdc-b1480b7be7b7, kind: Instance { id: c6a8d293-c1ab-4f47-8031-f4eef0d226b0 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 254, 145, 89])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(5333119), primary: true, slot: 0 }], source_nat: SourceNatConfig { ip: 10.151.1.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.1.176), floating_ips: [], firewall_rules: [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1815e9a0-21e4-4ca5-afdc-b1480b7be7b7, kind: Instance { id: c6a8d293-c1ab-4f47-8031-f4eef0d226b0 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 254, 145, 89])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(5333119), primary: true, slot: 0 }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1815e9a0-21e4-4ca5-afdc-b1480b7be7b7, kind: Instance { id: c6a8d293-c1ab-4f47-8031-f4eef0d226b0 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 254, 145, 89])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(5333119), primary: true, slot: 0 }], filter_hosts: Some([Vpc(Vni(5333119))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 1815e9a0-21e4-4ca5-afdc-b1480b7be7b7, kind: Instance { id: c6a8d293-c1ab-4f47-8031-f4eef0d226b0 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 254, 145, 89])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(5333119), primary: true, slot: 0 }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-d195aefd226\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: d0e76d2b-2c2d-4fa5-b65f-d33bb26662b8, read_only_parent: Some(Volume { block_size: 512, id: e22a8cd6-5b7d-4e19-8268-372d04d938c1, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 670fe997-47c9-4248-9de8-4a9fdf55b094, key: Some(\"CtbdWQ4ppBxTkdQkeSSUojYrJWWDLDANyrbEi2bo5n0=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::13]:19001\", \"[fd00:1122:3344:101::16]:19002\", \"[fd00:1122:3344:101::1a]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: d0e76d2b-2c2d-4fa5-b65f-d33bb26662b8, key: Some(\"7C+xY8bGV1FC9OJKNGJAYZnviY0IGQ3JgwSlsPbxB6U=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::1a]:19002\", \"[fd00:1122:3344:101::16]:19003\", \"[fd00:1122:3344:101::12]:19001\"] } }] } }], cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAABIiS3aWNpYAACJLdpYAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAASIkt2ljaWAAAiS3aWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiJjNmE4ZDI5My1jMWFiLTRmNDctODAzMS1mNGVlZjBkMjI2YjAiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJSHErcSt5SldHRkp5NzlPaEU0TnY0RHBzWmMra1J1OEJrQWlrNllpY1didiJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\") }, instance_runtime: InstanceRuntimeState { propolis_id: Some(6e15222a-e90a-4b46-9b34-bf4432694f7f (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-06-26T05:44:18.416175222Z }, vmm_runtime: VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-06-26T05:44:18.359720Z }, propolis_addr: [fd00:1122:3344:101::1:0]:12400 }","propolis_id":"6e15222a-e90a-4b46-9b34-bf4432694f7f","instance_id":"c6a8d293-c1ab-4f47-8031-f4eef0d226b0"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:18.776782706Z","hostname":"buskin","pid":1555,"uri":"/instances/c6a8d293-c1ab-4f47-8031-f4eef0d226b0","method":"PUT","req_id":"ac91a428-4c9d-4799-a7d7-9ae46dad6707","remote_addr":"[fd00:1122:3344:101::a]:37737","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":16625,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:18.876273397Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:60406"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:18.878433228Z","hostname":"buskin","pid":1555,"uri":"/instances/c6a8d293-c1ab-4f47-8031-f4eef0d226b0/state","method":"PUT","req_id":"83d34db9-217e-4acd-9ef9-80e7f7afcf34","remote_addr":"[fd00:1122:3344:101::a]:60406","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":2079,"response_code":"200"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:18.902265684Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:393","port":"Port { inner: PortInner { name: \"opte6\", ip: 172.30.0.5, mac: MacAddr6([168, 64, 37, 254, 145, 89]), slot: 0, vni: Vni { inner: 5333119 }, gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.0.1 }, vnic: \"vopte6\" } }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:18.902354156Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Configuring new Omicron zone: oxz_propolis-server_6e15222a-e90a-4b46-9b34-bf4432694f7f","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:18.93938221Z","hostname":"buskin","pid":1555,"instance_id":"c6a8d293-c1ab-4f47-8031-f4eef0d226b0","component":"InstanceManager","file":"illumos-utils/src/zone.rs:315"} {"msg":"Installing Omicron zone: oxz_propolis-server_6e15222a-e90a-4b46-9b34-bf4432694f7f","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:18.960870811Z","hostname":"buskin","pid":1555,"instance_id":"c6a8d293-c1ab-4f47-8031-f4eef0d226b0","component":"InstanceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"running automatic periodic zone bundle cleanup","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:19.093418074Z","hostname":"buskin","pid":1555,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:300"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:19.093501456Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:19.135359432Z","hostname":"buskin","pid":1555,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:289","time_to_next_cleanup":"599.99999713s","next_cleanup":"Instant { tv_sec: 1782, tv_nsec: 675440177 }"} {"msg":"Profile for oxz_propolis-server_6e15222a-e90a-4b46-9b34-bf4432694f7f:\n\n\n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:20.331850539Z","hostname":"buskin","pid":1555,"instance_id":"c6a8d293-c1ab-4f47-8031-f4eef0d226b0","component":"InstanceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Zone booting","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:20.332002593Z","hostname":"buskin","pid":1555,"zone":"oxz_propolis-server_6e15222a-e90a-4b46-9b34-bf4432694f7f","instance_id":"c6a8d293-c1ab-4f47-8031-f4eef0d226b0","component":"InstanceManager","file":"illumos-utils/src/running_zone.rs:513"} {"msg":"wait for service Some(\"oxz_propolis-server_6e15222a-e90a-4b46-9b34-bf4432694f7f\") failed: Property not found. retry in 58.546318ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:44:20.526165213Z","hostname":"buskin","pid":1555,"zone":"oxz_propolis-server_6e15222a-e90a-4b46-9b34-bf4432694f7f","instance_id":"c6a8d293-c1ab-4f47-8031-f4eef0d226b0","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_6e15222a-e90a-4b46-9b34-bf4432694f7f\") failed: Property not found. retry in 110.367532ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:44:20.598957911Z","hostname":"buskin","pid":1555,"zone":"oxz_propolis-server_6e15222a-e90a-4b46-9b34-bf4432694f7f","instance_id":"c6a8d293-c1ab-4f47-8031-f4eef0d226b0","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_6e15222a-e90a-4b46-9b34-bf4432694f7f\") failed: Property not found. retry in 193.532441ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:44:20.729462616Z","hostname":"buskin","pid":1555,"zone":"oxz_propolis-server_6e15222a-e90a-4b46-9b34-bf4432694f7f","instance_id":"c6a8d293-c1ab-4f47-8031-f4eef0d226b0","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_6e15222a-e90a-4b46-9b34-bf4432694f7f\") failed: Property not found. retry in 598.676006ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:44:20.939319852Z","hostname":"buskin","pid":1555,"zone":"oxz_propolis-server_6e15222a-e90a-4b46-9b34-bf4432694f7f","instance_id":"c6a8d293-c1ab-4f47-8031-f4eef0d226b0","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_6e15222a-e90a-4b46-9b34-bf4432694f7f\") failed: Property not found. retry in 759.550098ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:44:21.547674617Z","hostname":"buskin","pid":1555,"zone":"oxz_propolis-server_6e15222a-e90a-4b46-9b34-bf4432694f7f","instance_id":"c6a8d293-c1ab-4f47-8031-f4eef0d226b0","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_6e15222a-e90a-4b46-9b34-bf4432694f7f\") failed: Property not found. retry in 691.70821ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:44:22.320088859Z","hostname":"buskin","pid":1555,"zone":"oxz_propolis-server_6e15222a-e90a-4b46-9b34-bf4432694f7f","instance_id":"c6a8d293-c1ab-4f47-8031-f4eef0d226b0","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_6e15222a-e90a-4b46-9b34-bf4432694f7f\") failed: Property not found. retry in 1.113005448s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:44:23.023838968Z","hostname":"buskin","pid":1555,"zone":"oxz_propolis-server_6e15222a-e90a-4b46-9b34-bf4432694f7f","instance_id":"c6a8d293-c1ab-4f47-8031-f4eef0d226b0","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_6e15222a-e90a-4b46-9b34-bf4432694f7f\") failed: Property not found. retry in 1.281840769s","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:44:24.146541653Z","hostname":"buskin","pid":1555,"zone":"oxz_propolis-server_6e15222a-e90a-4b46-9b34-bf4432694f7f","instance_id":"c6a8d293-c1ab-4f47-8031-f4eef0d226b0","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Started propolis in zone: oxz_propolis-server_6e15222a-e90a-4b46-9b34-bf4432694f7f","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:25.461739169Z","hostname":"buskin","pid":1555,"instance_id":"c6a8d293-c1ab-4f47-8031-f4eef0d226b0","component":"InstanceManager","file":"sled-agent/src/instance.rs:1427"} {"msg":"wait for service Some(\"oxz_propolis-server_6e15222a-e90a-4b46-9b34-bf4432694f7f\") failed: Property not found. retry in 29.894352ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:44:25.471490756Z","hostname":"buskin","pid":1555,"instance_id":"c6a8d293-c1ab-4f47-8031-f4eef0d226b0","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_6e15222a-e90a-4b46-9b34-bf4432694f7f\") failed: Property not found. retry in 100.958483ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:44:25.513614039Z","hostname":"buskin","pid":1555,"instance_id":"c6a8d293-c1ab-4f47-8031-f4eef0d226b0","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_6e15222a-e90a-4b46-9b34-bf4432694f7f\") failed: Property not found. retry in 165.323909ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:44:25.624318362Z","hostname":"buskin","pid":1555,"instance_id":"c6a8d293-c1ab-4f47-8031-f4eef0d226b0","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_6e15222a-e90a-4b46-9b34-bf4432694f7f\") failed: Property not found. retry in 236.045485ms","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:44:25.800648666Z","hostname":"buskin","pid":1555,"instance_id":"c6a8d293-c1ab-4f47-8031-f4eef0d226b0","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Propolis SMF service is online","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:26.047868134Z","hostname":"buskin","pid":1555,"instance_id":"c6a8d293-c1ab-4f47-8031-f4eef0d226b0","component":"InstanceManager","file":"sled-agent/src/instance.rs:1436"} {"msg":"Propolis HTTP server online","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:26.083411204Z","hostname":"buskin","pid":1555,"instance_id":"c6a8d293-c1ab-4f47-8031-f4eef0d226b0","component":"InstanceManager","file":"sled-agent/src/instance.rs:1450"} {"msg":"Sending ensure request to propolis: InstanceEnsureRequest { cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAABIiS3aWNpYAACJLdpYAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAASIkt2ljaWAAAiS3aWAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiJjNmE4ZDI5My1jMWFiLTRmNDctODAzMS1mNGVlZjBkMjI2YjAiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJSHErcSt5SldHRkp5NzlPaEU0TnY0RHBzWmMra1J1OEJrQWlrNllpY1didiJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\"), disks: [DiskRequest { device: \"nvme\", name: \"disk-d195aefd226\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: d0e76d2b-2c2d-4fa5-b65f-d33bb26662b8, read_only_parent: Some(Volume { block_size: 512, id: e22a8cd6-5b7d-4e19-8268-372d04d938c1, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 670fe997-47c9-4248-9de8-4a9fdf55b094, key: Some(\"CtbdWQ4ppBxTkdQkeSSUojYrJWWDLDANyrbEi2bo5n0=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::13]:19001\", \"[fd00:1122:3344:101::16]:19002\", \"[fd00:1122:3344:101::1a]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: d0e76d2b-2c2d-4fa5-b65f-d33bb26662b8, key: Some(\"7C+xY8bGV1FC9OJKNGJAYZnviY0IGQ3JgwSlsPbxB6U=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::1a]:19002\", \"[fd00:1122:3344:101::16]:19003\", \"[fd00:1122:3344:101::12]:19001\"] } }] } }], migrate: None, nics: [NetworkInterfaceRequest { name: \"vopte6\", slot: Slot(0) }], properties: InstanceProperties { bootrom_id: 00000000-0000-0000-0000-000000000000, description: \"Test description\", id: c6a8d293-c1ab-4f47-8031-f4eef0d226b0, image_id: 00000000-0000-0000-0000-000000000000, memory: 1024, metadata: InstanceMetadata { project_id: 1efb3ce2-5910-46a8-8065-63018354c09d, silo_id: 6334f5b4-1266-4926-aaf8-bf8279271639 }, name: \"localshark\", vcpus: 2 } }","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:26.084753905Z","hostname":"buskin","pid":1555,"instance_id":"c6a8d293-c1ab-4f47-8031-f4eef0d226b0","component":"InstanceManager","file":"sled-agent/src/instance.rs:689"} {"msg":"result of instance_ensure call is Ok(InstanceEnsureResponse { migrate: None })","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:26.237285834Z","hostname":"buskin","pid":1555,"instance_id":"c6a8d293-c1ab-4f47-8031-f4eef0d226b0","component":"InstanceManager","file":"sled-agent/src/instance.rs:691"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Starting), migration_status: NoMigration, time: 2024-06-26T05:44:26.237714004Z }","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:26.237738144Z","hostname":"buskin","pid":1555,"instance_id":"c6a8d293-c1ab-4f47-8031-f4eef0d226b0","component":"InstanceManager","file":"sled-agent/src/instance.rs:573"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:26.237756585Z","hostname":"buskin","pid":1555,"instance_id":"c6a8d293-c1ab-4f47-8031-f4eef0d226b0","component":"InstanceManager","file":"sled-agent/src/instance.rs:595","new_vmm_state":"VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-06-26T05:44:26.237714004Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(6e15222a-e90a-4b46-9b34-bf4432694f7f (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-06-26T05:44:18.416175222Z }","propolis_id":"6e15222a-e90a-4b46-9b34-bf4432694f7f"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:26.237778255Z","hostname":"buskin","pid":1555,"instance_id":"c6a8d293-c1ab-4f47-8031-f4eef0d226b0","component":"InstanceManager","file":"sled-agent/src/instance.rs:492","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(6e15222a-e90a-4b46-9b34-bf4432694f7f (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-06-26T05:44:18.416175222Z }, propolis_id: 6e15222a-e90a-4b46-9b34-bf4432694f7f (propolis), vmm_state: VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-06-26T05:44:26.237714004Z }, migration_state: None }","instance_id":"c6a8d293-c1ab-4f47-8031-f4eef0d226b0"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Running), migration_status: NoMigration, time: 2024-06-26T05:44:26.321245203Z }","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:26.321264233Z","hostname":"buskin","pid":1555,"instance_id":"c6a8d293-c1ab-4f47-8031-f4eef0d226b0","component":"InstanceManager","file":"sled-agent/src/instance.rs:573"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:26.321284234Z","hostname":"buskin","pid":1555,"instance_id":"c6a8d293-c1ab-4f47-8031-f4eef0d226b0","component":"InstanceManager","file":"sled-agent/src/instance.rs:595","new_vmm_state":"VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-06-26T05:44:26.321245203Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(6e15222a-e90a-4b46-9b34-bf4432694f7f (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-06-26T05:44:18.416175222Z }","propolis_id":"6e15222a-e90a-4b46-9b34-bf4432694f7f"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:26.321311074Z","hostname":"buskin","pid":1555,"instance_id":"c6a8d293-c1ab-4f47-8031-f4eef0d226b0","component":"InstanceManager","file":"sled-agent/src/instance.rs:492","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(6e15222a-e90a-4b46-9b34-bf4432694f7f (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-06-26T05:44:18.416175222Z }, propolis_id: 6e15222a-e90a-4b46-9b34-bf4432694f7f (propolis), vmm_state: VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-06-26T05:44:26.321245203Z }, migration_state: None }","instance_id":"c6a8d293-c1ab-4f47-8031-f4eef0d226b0"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:32.383723312Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:64743"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:32.384465929Z","hostname":"buskin","pid":1555,"uri":"/instances/c6a8d293-c1ab-4f47-8031-f4eef0d226b0/state","method":"GET","req_id":"b1b07703-a807-4c96-b572-7bab8bafe75d","remote_addr":"[fd00:1122:3344:101::c]:64743","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":703,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:32.38492199Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:59433"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:32.385207287Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"f3e67b1d-93c3-4dd5-b5dd-fa7b368d1d24","remote_addr":"[fd00:1122:3344:101::c]:59433","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":199,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:32.484257668Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:54371"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:32.537294375Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"de1bb11f-658e-4aaf-97fd-fb08f1259d90","remote_addr":"[fd00:1122:3344:101::c]:54371","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":52985,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:32.537688044Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"cef44517-9061-499c-a9d6-c8596bc3de77","remote_addr":"[fd00:1122:3344:101::c]:54371","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":166,"response_code":"200"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:32.655200735Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:831"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:32.660896788Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:38826"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:32.661178015Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"cede86c0-2566-4146-a45c-867cd4dbc176","remote_addr":"[fd00:1122:3344:101::a]:38826","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":217,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:32.706778469Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:38744"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:32.706928242Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"16880520-2c5b-47b0-91f5-ea7975924597","remote_addr":"[fd00:1122:3344:101::b]:38744","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":127,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:32.734608588Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:38884"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:32.734770122Z","hostname":"buskin","pid":1555,"uri":"/instances/c6a8d293-c1ab-4f47-8031-f4eef0d226b0/state","method":"GET","req_id":"06b9f276-12e9-4082-94b5-9635c7e4516f","remote_addr":"[fd00:1122:3344:101::a]:38884","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":129,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:32.759265933Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:56977"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:32.759458678Z","hostname":"buskin","pid":1555,"uri":"/instances/c6a8d293-c1ab-4f47-8031-f4eef0d226b0/state","method":"GET","req_id":"36f3f5ef-c845-40af-a190-4e9cec21d0df","remote_addr":"[fd00:1122:3344:101::b]:56977","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":136,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:32.828312455Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:45911"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:32.875031474Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"456da5c2-ef7e-4a16-90d0-d430b9545e07","remote_addr":"[fd00:1122:3344:101::a]:45911","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":46677,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:32.875351862Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"57b66f38-b79c-42fa-8b0b-9f797b1bf331","remote_addr":"[fd00:1122:3344:101::a]:45911","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":129,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:32.889974163Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::b]:50177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:32.936235122Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"c9b7b157-fdd1-476d-a60f-04e7afec0599","remote_addr":"[fd00:1122:3344:101::b]:50177","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":46177,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:32.936486108Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"af155f9f-4ced-4a84-b14d-d7be80e1524a","remote_addr":"[fd00:1122:3344:101::b]:50177","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":102,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:36.005111695Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:37018"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:36.005694108Z","hostname":"buskin","pid":1555,"uri":"/instances/c6a8d293-c1ab-4f47-8031-f4eef0d226b0/state","method":"PUT","req_id":"51a8419b-e87c-4d39-9dd3-00363a3db356","remote_addr":"[fd00:1122:3344:101::a]:37018","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":553,"response_code":"200"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Stopping), migration_status: NoMigration, time: 2024-06-26T05:44:36.005803011Z }","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:36.005825741Z","hostname":"buskin","pid":1555,"instance_id":"c6a8d293-c1ab-4f47-8031-f4eef0d226b0","component":"InstanceManager","file":"sled-agent/src/instance.rs:573"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:36.005844072Z","hostname":"buskin","pid":1555,"instance_id":"c6a8d293-c1ab-4f47-8031-f4eef0d226b0","component":"InstanceManager","file":"sled-agent/src/instance.rs:595","new_vmm_state":"VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-06-26T05:44:36.005803011Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(6e15222a-e90a-4b46-9b34-bf4432694f7f (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-06-26T05:44:18.416175222Z }","propolis_id":"6e15222a-e90a-4b46-9b34-bf4432694f7f"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:36.005870792Z","hostname":"buskin","pid":1555,"instance_id":"c6a8d293-c1ab-4f47-8031-f4eef0d226b0","component":"InstanceManager","file":"sled-agent/src/instance.rs:492","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(6e15222a-e90a-4b46-9b34-bf4432694f7f (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-06-26T05:44:18.416175222Z }, propolis_id: 6e15222a-e90a-4b46-9b34-bf4432694f7f (propolis), vmm_state: VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-06-26T05:44:36.005803011Z }, migration_state: None }","instance_id":"c6a8d293-c1ab-4f47-8031-f4eef0d226b0"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Destroyed), migration_status: NoMigration, time: 2024-06-26T05:44:36.052189893Z }","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:36.052217144Z","hostname":"buskin","pid":1555,"instance_id":"c6a8d293-c1ab-4f47-8031-f4eef0d226b0","component":"InstanceManager","file":"sled-agent/src/instance.rs:573"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:36.052244214Z","hostname":"buskin","pid":1555,"instance_id":"c6a8d293-c1ab-4f47-8031-f4eef0d226b0","component":"InstanceManager","file":"sled-agent/src/instance.rs:595","new_vmm_state":"VmmRuntimeState { state: Destroyed, gen: Generation(6), time_updated: 2024-06-26T05:44:36.052189893Z }","new_instance_state":"InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-06-26T05:44:36.052189893Z }","propolis_id":"6e15222a-e90a-4b46-9b34-bf4432694f7f"} {"msg":"terminating VMM that has exited","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:36.052260805Z","hostname":"buskin","pid":1555,"instance_id":"c6a8d293-c1ab-4f47-8031-f4eef0d226b0","component":"InstanceManager","file":"sled-agent/src/instance.rs:610","instance_id":"c6a8d293-c1ab-4f47-8031-f4eef0d226b0"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:36.052267965Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:36.05248912Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:381"} {"msg":"creating zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:36.052532381Z","hostname":"buskin","pid":1555,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:437","context":"ZoneBundleContext { storage_dirs: [\"/pool/int/968bbfe8-8122-4a85-96dd-954188a2f89d/debug/bundle/zone\", \"/pool/int/e8396e20-0493-4030-b55b-5c9e59a237cf/debug/bundle/zone\"], cause: TerminatedInstance, extra_log_dirs: [\"/pool/ext/2a12c7dc-fee7-4f85-ab9e-afd52f78b31f/crypt/debug\", \"/pool/ext/5e9c007a-1be0-4b01-8d85-c9b89975656a/crypt/debug\", \"/pool/ext/d4ac128c-bdc1-48a9-b5af-10262f74baf6/crypt/debug\", \"/pool/ext/4f34febf-41f3-4c60-ba39-4554d702f4af/crypt/debug\", \"/pool/ext/9ffbf4d7-0dbd-45c3-94b3-17c5d304c2bd/crypt/debug\", \"/pool/ext/bd06e234-5970-4365-a0e5-6cd29811daea/crypt/debug\", \"/pool/ext/e1e381d6-18e9-425e-b2e0-7789234ef986/crypt/debug\", \"/pool/ext/4031e1c2-96e5-4572-96c7-90799cc1b09a/crypt/debug\", \"/pool/ext/cdd9c631-6644-4cda-aca6-05f7f580f308/crypt/debug\"] }","zone_name":"oxz_propolis-server_6e15222a-e90a-4b46-9b34-bf4432694f7f"} {"msg":"finished zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:36.677265966Z","hostname":"buskin","pid":1555,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:1230","metadata":"ZoneBundleMetadata { id: ZoneBundleId { zone_name: \"oxz_propolis-server_6e15222a-e90a-4b46-9b34-bf4432694f7f\", bundle_id: 801dc1c7-d5d3-490c-87dc-948db2aaa59a }, time_created: 2024-06-26T05:44:36.052812268Z, version: 0, cause: TerminatedInstance }"} {"msg":"Halting and removing zone: oxz_propolis-server_6e15222a-e90a-4b46-9b34-bf4432694f7f","v":0,"name":"SledAgent","level":40,"time":"2024-06-26T05:44:36.677355429Z","hostname":"buskin","pid":1555,"instance_id":"c6a8d293-c1ab-4f47-8031-f4eef0d226b0","component":"InstanceManager","file":"sled-agent/src/instance.rs:784"} {"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:38.426646963Z","hostname":"buskin","pid":1555,"instance_id":"c6a8d293-c1ab-4f47-8031-f4eef0d226b0","component":"InstanceManager","file":"illumos-utils/src/zone.rs:269"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:38.450660233Z","hostname":"buskin","pid":1555,"instance_id":"c6a8d293-c1ab-4f47-8031-f4eef0d226b0","component":"InstanceManager","file":"sled-agent/src/instance.rs:492","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-06-26T05:44:36.052189893Z }, propolis_id: 6e15222a-e90a-4b46-9b34-bf4432694f7f (propolis), vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-06-26T05:44:38.450596492Z }, migration_state: None }","instance_id":"c6a8d293-c1ab-4f47-8031-f4eef0d226b0"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:38.557313802Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::c]:43645"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:38.557610559Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"4617d37d-fe80-49f7-86f6-ae015f1d5cd1","remote_addr":"[fd00:1122:3344:101::c]:43645","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":246,"response_code":"200"} {"msg":"Mapping virtual NIC to physical host","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:38.557863765Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:631","mapping":"VirtualNetworkInterfaceHost { virtual_ip: 172.30.0.5, virtual_mac: MacAddr(MacAddr6([168, 64, 37, 254, 145, 89])), physical_host_ip: fd00:1122:3344:101::1, vni: Vni(5333119) }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:38.557900216Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"PUT","req_id":"45f7e15d-0392-47c3-8784-5ad981b4d604","remote_addr":"[fd00:1122:3344:101::c]:43645","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":120,"response_code":"204"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:38.837827737Z","hostname":"buskin","pid":1555,"instance_id":"c6a8d293-c1ab-4f47-8031-f4eef0d226b0","component":"InstanceManager","file":"sled-agent/src/instance.rs:492","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-06-26T05:44:36.052189893Z }, propolis_id: 6e15222a-e90a-4b46-9b34-bf4432694f7f (propolis), vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-06-26T05:44:38.450596492Z }, migration_state: None }","instance_id":"c6a8d293-c1ab-4f47-8031-f4eef0d226b0"} {"msg":"State monitoring task complete","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:38.84139466Z","hostname":"buskin","pid":1555,"instance_id":"c6a8d293-c1ab-4f47-8031-f4eef0d226b0","component":"InstanceManager","file":"sled-agent/src/instance.rs:727"} {"msg":"Failed to publish state to Nexus, will not retry: Notification(Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"49c43334-847a-42ac-b5d8-fd490d145d5a\", \"content-length\": \"177\", \"date\": \"Wed, 26 Jun 2024 05:44:38 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: instance with id \\\"c6a8d293-c1ab-4f47-8031-f4eef0d226b0\\\"\", request_id: \"49c43334-847a-42ac-b5d8-fd490d145d5a\" })","v":0,"name":"SledAgent","level":50,"time":"2024-06-26T05:44:38.881454605Z","hostname":"buskin","pid":1555,"instance_id":"c6a8d293-c1ab-4f47-8031-f4eef0d226b0","component":"InstanceManager","file":"sled-agent/src/instance.rs:559","instance_id":"c6a8d293-c1ab-4f47-8031-f4eef0d226b0"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:38.989383783Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fd00:1122:3344:101::a]:48571"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:38.989635289Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"9dfb60ce-9122-4b46-abf2-4686c6f01a30","remote_addr":"[fd00:1122:3344:101::a]:48571","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":224,"response_code":"200"} {"msg":"Clearing mapping of virtual NIC to physical host","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:38.990612672Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:671","mapping":"VirtualNetworkInterfaceHost { virtual_ip: 172.30.0.5, virtual_mac: MacAddr(MacAddr6([168, 64, 37, 254, 145, 89])), physical_host_ip: fd00:1122:3344:101::1, vni: Vni(5333119) }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:38.991160014Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"DELETE","req_id":"d923da63-8c9e-407d-a6bf-d28b74739c41","remote_addr":"[fd00:1122:3344:101::a]:48571","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":1283,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:45.750305669Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:766","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:61157"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-06-26T05:44:45.750395931Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"fdd65fe5-362d-4c20-a67d-9839c62b8202","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:61157","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0cd0e82/dropshot/src/server.rs:878","latency_us":83,"response_code":"200"}