[ Jul 25 05:44:00 Enabled. ] [ Jul 25 05:44:00 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/sled-agent/sled-agent run /opt/oxide/sled-agent/pkg/config.toml &"). ] [ Jul 25 05:44:00 Method "start" exited with status 0. ] [ Jul 25 05:44:00 Rereading configuration. ] [ Jul 25 05:44:00 No 'refresh' method defined. Treating as :true. ] note: configured to log to "/dev/stdout" {"msg":"Ensuring contract reaper thread","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:00.710832346Z","hostname":"buskin","pid":1555,"file":"illumos-utils/src/running_zone.rs:110"} {"msg":"finding links [\"net0\", \"net1\"]","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:00.711049831Z","hostname":"buskin","pid":1555,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:255"} {"msg":"Starting mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:00.764735626Z","hostname":"buskin","pid":1555,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:262","addr-objs":"[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]"} {"msg":"Importing mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:00.764784167Z","hostname":"buskin","pid":1555,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:42","path":"/opt/oxide/mg-ddm/pkg/ddm/manifest.xml"} {"msg":"Setting mg-ddm interfaces","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:00.880336381Z","hostname":"buskin","pid":1555,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:50","interfaces":"(\"net0/ll\" \"net1/ll\")"} {"msg":"Enabling mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:00.889314524Z","hostname":"buskin","pid":1555,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:56"} {"msg":"Ensuring zfs key directory exists","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:00.897677282Z","hostname":"buskin","pid":1555,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:272","path":"/var/run/oxide/"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:01.497039515Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fdb0:18c0:4d0d:9fb2::, width: 64 }"} {"msg":"Starting KeyManager","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:01.640178374Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:125"} {"msg":"Starting StorageManager","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:01.640214295Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:137"} {"msg":"Starting StorageMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:01.640309247Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:150"} {"msg":"Starting HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:01.640388549Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:172","nongimlet_observed_disks":"[]","sled_mode":"Scrimlet { asic: SoftNpuZone }"} {"msg":"DumpSetup poll loop started.","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:01.64042477Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:520"} {"msg":"Creating HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:01.64043093Z","hostname":"buskin","pid":1555,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:718"} {"msg":"Generated i86pc baseboard Pc { identifier: \"buskin\", model: \"i86pc\" }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:01.643508063Z","hostname":"buskin","pid":1555,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:625"} {"msg":"Device is not a Gimlet (i86pc), proceeding with null hardware view","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:01.643514503Z","hostname":"buskin","pid":1555,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:767"} {"msg":"Starting HardwareMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:01.643546724Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:186"} {"msg":"Monitoring for hardware updates","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:01.643552894Z","hostname":"buskin","pid":1555,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:826"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:01.643587055Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:242","vdev":"/scratch/m2_0.vdev"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:01.643626406Z","hostname":"buskin","pid":1555,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:271","disks":"{}","underlay_network_info":"None"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:01.643755439Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:01.64378649Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:01.64379991Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:127","path":"/scratch/m2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:01.654838992Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:361"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:01.654878233Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:372","id":"6d2aa15e-618e-402e-b368-cf27fd8def68 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:01.750458253Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: 6d2aa15e-618e-402e-b368-cf27fd8def68 (zpool), kind: Internal }"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:01.96047646Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:01.960509281Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:242","vdev":"/scratch/m2_1.vdev"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:01.960514641Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:601","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:01.960528971Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:622","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:01.960536411Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:01.960556402Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:01.960561522Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:127","path":"/scratch/m2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:01.969317179Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:361"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:01.96934606Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:372","id":"75c788e1-45ef-4fcf-958d-db07331bef52 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.056384907Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: 75c788e1-45ef-4fcf-958d-db07331bef52 (zpool), kind: Internal }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.258873005Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:242","vdev":"/scratch/u2_0.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.258908686Z","hostname":"buskin","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 6d2aa15e-618e-402e-b368-cf27fd8def68 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 75c788e1-45ef-4fcf-958d-db07331bef52 (zpool), kind: Internal } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.258941767Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.258950547Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.258956527Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.258976378Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.259015088Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:242","vdev":"/scratch/u2_1.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.259025049Z","hostname":"buskin","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 6d2aa15e-618e-402e-b368-cf27fd8def68 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 75c788e1-45ef-4fcf-958d-db07331bef52 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.259039579Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.259044639Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.259049189Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.2590625Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.25906683Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:242","vdev":"/scratch/u2_2.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.25908104Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.25908622Z","hostname":"buskin","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 6d2aa15e-618e-402e-b368-cf27fd8def68 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 75c788e1-45ef-4fcf-958d-db07331bef52 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.25910227Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.259108531Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.259113791Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.259117521Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:242","vdev":"/scratch/u2_3.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.259121851Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.259126381Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.259130131Z","hostname":"buskin","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 6d2aa15e-618e-402e-b368-cf27fd8def68 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 75c788e1-45ef-4fcf-958d-db07331bef52 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.259153332Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.259157832Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.259162302Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:242","vdev":"/scratch/u2_4.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.259166612Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.259171032Z","hostname":"buskin","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 6d2aa15e-618e-402e-b368-cf27fd8def68 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 75c788e1-45ef-4fcf-958d-db07331bef52 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.259188172Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.259192443Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.259196733Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.259200683Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:242","vdev":"/scratch/u2_5.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.259205423Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.259208943Z","hostname":"buskin","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 6d2aa15e-618e-402e-b368-cf27fd8def68 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 75c788e1-45ef-4fcf-958d-db07331bef52 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.259234944Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.259239534Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.259248304Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.259251934Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:242","vdev":"/scratch/u2_6.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.259256324Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.259260744Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.259264654Z","hostname":"buskin","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 6d2aa15e-618e-402e-b368-cf27fd8def68 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 75c788e1-45ef-4fcf-958d-db07331bef52 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.259292625Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.259297975Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.259301245Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:242","vdev":"/scratch/u2_7.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.259305405Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.259310086Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.259314035Z","hostname":"buskin","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 6d2aa15e-618e-402e-b368-cf27fd8def68 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 75c788e1-45ef-4fcf-958d-db07331bef52 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.259346426Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.259351307Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.259354696Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:242","vdev":"/scratch/u2_8.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.259359057Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.259362437Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.259366687Z","hostname":"buskin","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 6d2aa15e-618e-402e-b368-cf27fd8def68 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 75c788e1-45ef-4fcf-958d-db07331bef52 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.259395228Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.259399677Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Waiting for boot disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.259402988Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:96"} {"msg":"Found boot disk DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.259411918Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:98"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.259417958Z","hostname":"buskin","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 6d2aa15e-618e-402e-b368-cf27fd8def68 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 75c788e1-45ef-4fcf-958d-db07331bef52 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.259446999Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.259451509Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.259454819Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Starting Bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.259458129Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:210"} {"msg":"No ledger in /pool/int/6d2aa15e-618e-402e-b368-cf27fd8def68/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.259470319Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/75c788e1-45ef-4fcf-958d-db07331bef52/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.25949828Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found. Loading Fsm::State as Uninitialized","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.25950384Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:83"} {"msg":"No ledger in /pool/int/6d2aa15e-618e-402e-b368-cf27fd8def68/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.25951331Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/75c788e1-45ef-4fcf-958d-db07331bef52/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.25952162Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found for network config","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.259528851Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:146"} {"msg":"Starting Bootstore DDMD poller","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.259552211Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:215"} {"msg":"Starting ZoneBundler related tasks","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.259556601Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:230"} {"msg":"Creating ServiceManager","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.270219965Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:719"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.270313427Z","hostname":"buskin","pid":1555,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:292","time_to_next_cleanup":"599.999906668s","next_cleanup":"Instant { tv_sec: 6204, tv_nsec: 880097656 }"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.270348268Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3665"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.270372468Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"No ledger in /pool/int/6d2aa15e-618e-402e-b368-cf27fd8def68/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.27045024Z","hostname":"buskin","pid":1555,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/75c788e1-45ef-4fcf-958d-db07331bef52/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.27046239Z","hostname":"buskin","pid":1555,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.27381607Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:205"} {"msg":"Started listening","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.273891312Z","hostname":"buskin","pid":1555,"component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:44","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Injecting RSS configuration: RackInitializeRequest {\n trust_quorum_peers: None,\n bootstrap_discovery: OnlyOurs,\n ntp_servers: [\n \"0.pool.ntp.org\",\n ],\n dns_servers: [\n 1.1.1.1,\n 9.9.9.9,\n ],\n internal_services_ip_pool_ranges: [\n V4(\n Ipv4Range {\n first: 10.151.1.164,\n last: 10.151.1.173,\n },\n ),\n ],\n external_dns_ips: [\n 10.151.1.164,\n 10.151.1.165,\n ],\n external_dns_zone_name: \"oxide.test\",\n external_certificates: \"\",\n recovery_silo: RecoverySiloConfig {\n silo_name: Name(\n \"recovery\",\n ),\n user_name: UserId(\n \"recovery\",\n ),\n user_password_hash: NewPasswordHash(\n PasswordHashString {\n string: \"$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY\",\n encoding: B64,\n },\n ),\n },\n rack_network_config: RackNetworkConfigV2 {\n rack_subnet: Ipv6Net {\n addr: fd00:1122:3344:100::,\n width: 56,\n },\n infra_ip_first: 10.151.1.174,\n infra_ip_last: 10.151.1.174,\n ports: [\n PortConfigV2 {\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n nexthop: 10.151.1.100,\n vlan_id: None,\n },\n ],\n addresses: [\n UplinkAddressConfig {\n address: V4(\n Ipv4Net {\n addr: 10.151.1.174,\n width: 24,\n },\n ),\n vlan_id: None,\n },\n ],\n switch: Switch0,\n port: \"qsfp0\",\n uplink_port_speed: Speed40G,\n uplink_port_fec: None,\n bgp_peers: [],\n autoneg: false,\n },\n ],\n bgp: [],\n bfd: [],\n },\n allowed_source_ips: Any,\n}","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.274109987Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1014"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.274116897Z","hostname":"buskin","pid":1555,"component":"DnsResolver","component":"RSS","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.274182829Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"No ledger in /pool/int/6d2aa15e-618e-402e-b368-cf27fd8def68/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.27422931Z","hostname":"buskin","pid":1555,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/75c788e1-45ef-4fcf-958d-db07331bef52/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.27425117Z","hostname":"buskin","pid":1555,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"RSS configuration has not been fully applied yet","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.27426001Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1074"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.274285901Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"No ledger in /pool/int/6d2aa15e-618e-402e-b368-cf27fd8def68/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.274310552Z","hostname":"buskin","pid":1555,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/75c788e1-45ef-4fcf-958d-db07331bef52/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.274319852Z","hostname":"buskin","pid":1555,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"Creating new allocation plan","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.274324772Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1115"} {"msg":"Serializing plan","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.274329232Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:156"} {"msg":"Creating plan for the sled at fdb0:18c0:4d0d:9fb2::1","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.274333772Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:129"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.274364283Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Writing ledger to /pool/int/6d2aa15e-618e-402e-b368-cf27fd8def68/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.274392434Z","hostname":"buskin","pid":1555,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/75c788e1-45ef-4fcf-958d-db07331bef52/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.274739222Z","hostname":"buskin","pid":1555,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Sled plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.274901206Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:176"} {"msg":"Writing Rack Network Configuration to bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.274921546Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1148"} {"msg":"Attempting to update network config with generation: 1, current_generation: 0","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.274979098Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/6d2aa15e-618e-402e-b368-cf27fd8def68/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.274993828Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/75c788e1-45ef-4fcf-958d-db07331bef52/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.275164632Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 1","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.275297465Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"Received initialization request from RSS","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.275406088Z","hostname":"buskin","pid":1555,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:217","target_sled":"[fdb0:18c0:4d0d:9fb2::1]:12346","request":"StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: 8fa4bc78-1026-41f0-b09d-c67224071376, rack_id: b4c6410d-62e1-4369-a8c0-f8e6399dd808, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }"} {"msg":"Accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.275557861Z","hostname":"buskin","pid":1555,"remote_addr":"[fdb0:18c0:4d0d:9fb2::1]:57030","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:66"} {"msg":"Loading Sled Agent: StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: 8fa4bc78-1026-41f0-b09d-c67224071376, rack_id: b4c6410d-62e1-4369-a8c0-f8e6399dd808, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.275752506Z","hostname":"buskin","pid":1555,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:359"} {"msg":"KeyManager: using hardcoded secret retriever","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.275757396Z","hostname":"buskin","pid":1555,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:374"} {"msg":"setting up sled agent server","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.275854718Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/server.rs:44"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.275886989Z","hostname":"buskin","pid":1555,"component":"DnsResolver","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.275891199Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:101::, width: 64 }"} {"msg":"Received KeyManagerReady","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.275896239Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Loading storage config","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.27590002Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:572"} {"msg":"No ledger in /pool/int/6d2aa15e-618e-402e-b368-cf27fd8def68/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.27590991Z","hostname":"buskin","pid":1555,"request":"load_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/75c788e1-45ef-4fcf-958d-db07331bef52/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.275947821Z","hostname":"buskin","pid":1555,"request":"load_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger of physical disks exists","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.275957401Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:561"} {"msg":"KeyManager ready, but no ledger detected","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.275961171Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:589"} {"msg":"automatically managing disks","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.275966141Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:393"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.276022882Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:374"} {"msg":"Successfully synchronized disks without error","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.276035133Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:479","result":"DisksManagementResult { status: [] }"} {"msg":"Enabling switch zone (new)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.27761466Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3899"} {"msg":"Starting switch zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.277661021Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:4272"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.308116165Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Configuring new Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.328880238Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.345986764Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"SledAgent::new(..) starting","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.392953909Z","hostname":"buskin","pid":1555,"sled_id":"8fa4bc78-1026-41f0-b09d-c67224071376","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:365"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.39301013Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Requested swap device of size 64 GiB","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.393016881Z","hostname":"buskin","pid":1555,"sled_id":"8fa4bc78-1026-41f0-b09d-c67224071376","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:377"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.393023081Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 114.258176ms","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.396205416Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"32c6bba2-2f6a-4a9a-ac7a-4e82ed3164a6\", \"content-length\": \"124\", \"date\": \"Thu, 25 Jul 2024 05:44:01 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"32c6bba2-2f6a-4a9a-ac7a-4e82ed3164a6\" }"} {"msg":"attempting to create encrypted zvol: name=\"oxi_6d2aa15e-618e-402e-b368-cf27fd8def68/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.407482194Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/swap_device.rs:153"} {"msg":"successfully created encrypted zvol: name=\"oxi_6d2aa15e-618e-402e-b368-cf27fd8def68/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.431555936Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/swap_device.rs:249"} {"msg":"adding swap device: swapname=\"/dev/zvol/dsk/oxi_6d2aa15e-618e-402e-b368-cf27fd8def68/swap\"","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.431597147Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/swap_device.rs:96"} {"msg":"Mounting backing filesystems","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.437421485Z","hostname":"buskin","pid":1555,"sled_id":"8fa4bc78-1026-41f0-b09d-c67224071376","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:392"} {"msg":"Processing fmd","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.437435275Z","hostname":"buskin","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:126"} {"msg":"Ensuring dataset oxi_6d2aa15e-618e-402e-b368-cf27fd8def68/backing/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.437440876Z","hostname":"buskin","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:136"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.511160536Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Stopping service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.511650078Z","hostname":"buskin","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:170"} {"msg":"Failed enable ddm stats (will retry after 100.741553ms","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.514195718Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"4aaa740d-1eec-4bce-8079-c1434d2c2a92\", \"content-length\": \"124\", \"date\": \"Thu, 25 Jul 2024 05:44:02 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"4aaa740d-1eec-4bce-8079-c1434d2c2a92\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.616299783Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 91.514047ms","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.616812105Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"49e761db-b21b-4cdb-841d-4f56eac50283\", \"content-length\": \"124\", \"date\": \"Thu, 25 Jul 2024 05:44:02 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"49e761db-b21b-4cdb-841d-4f56eac50283\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.708693377Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 214.652667ms","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.709151418Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ef7941a1-220e-4365-a1a5-9a0df00b48c0\", \"content-length\": \"124\", \"date\": \"Thu, 25 Jul 2024 05:44:02 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"ef7941a1-220e-4365-a1a5-9a0df00b48c0\" }"} {"msg":"Mounting oxi_6d2aa15e-618e-402e-b368-cf27fd8def68/backing/fmd on /var/fm/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.731078178Z","hostname":"buskin","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:178"} {"msg":"Ensuring directory /var/fm/fmd/rsrc","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.744685861Z","hostname":"buskin","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Ensuring directory /var/fm/fmd/ckpt","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.744776904Z","hostname":"buskin","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Ensuring directory /var/fm/fmd/xprt","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.744803244Z","hostname":"buskin","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Starting service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.744840205Z","hostname":"buskin","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:192"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.925618528Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 124.25303ms","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:02.926314635Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a088797f-8bcf-40b8-ac5b-4f8eb1851afe\", \"content-length\": \"124\", \"date\": \"Thu, 25 Jul 2024 05:44:02 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a088797f-8bcf-40b8-ac5b-4f8eb1851afe\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:03.051164189Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 346.946173ms","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:03.051896466Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d36e60cc-f22d-45e4-9d39-ec187268c576\", \"content-length\": \"124\", \"date\": \"Thu, 25 Jul 2024 05:44:02 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d36e60cc-f22d-45e4-9d39-ec187268c576\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:03.399998442Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 275.491254ms","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:03.400687299Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e9776b26-434b-4ed0-977c-e01509241d2d\", \"content-length\": \"124\", \"date\": \"Thu, 25 Jul 2024 05:44:02 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"e9776b26-434b-4ed0-977c-e01509241d2d\" }"} {"msg":"using '[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]' as data links for xde driver","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:03.467371562Z","hostname":"buskin","pid":1555,"sled_id":"8fa4bc78-1026-41f0-b09d-c67224071376","component":"SledAgent","file":"illumos-utils/src/opte/illumos.rs:90"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:03.467972436Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:57429","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:205","component":"producer-server","component":"MetricsManager","sled_id":"8fa4bc78-1026-41f0-b09d-c67224071376","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:205"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:03.467991517Z","hostname":"buskin","pid":1555,"component":"internal-dns-resolver","file":"internal-dns/src/resolver.rs:60","component":"producer-server","component":"MetricsManager","sled_id":"8fa4bc78-1026-41f0-b09d-c67224071376","component":"SledAgent","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"starting oximeter metric producer server","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:03.468044028Z","hostname":"buskin","pid":1555,"file":"oximeter/producer/src/lib.rs:283","component":"producer-server","component":"MetricsManager","sled_id":"8fa4bc78-1026-41f0-b09d-c67224071376","component":"SledAgent","file":"oximeter/producer/src/lib.rs:283","interval":"30s","address":"[fd00:1122:3344:101::1]:57429","producer_id":"8fa4bc78-1026-41f0-b09d-c67224071376"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:03.474485621Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","component":"MetricsManager","sled_id":"8fa4bc78-1026-41f0-b09d-c67224071376","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"342.575265ms"} {"msg":"50% of 102952026112 physical ram = 51476013056 bytes)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:03.51442868Z","hostname":"buskin","pid":1555,"component":"VmmReservoirManager","sled_id":"8fa4bc78-1026-41f0-b09d-c67224071376","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:267"} {"msg":"Setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:03.51445461Z","hostname":"buskin","pid":1555,"component":"VmmReservoirManager","sled_id":"8fa4bc78-1026-41f0-b09d-c67224071376","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:275"} {"msg":"sled agent started","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:03.514524412Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:902","underlay_address":"fd00:1122:3344:101::1"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:03.514532672Z","hostname":"buskin","pid":1555,"component":"DnsResolver","component":"ServiceManager","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:03.515404493Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:205"} {"msg":"Sled Agent loaded; recording configuration","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:03.515430283Z","hostname":"buskin","pid":1555,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:429"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:03.515457994Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Writing ledger to /pool/int/6d2aa15e-618e-402e-b368-cf27fd8def68/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:03.515473455Z","hostname":"buskin","pid":1555,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/75c788e1-45ef-4fcf-958d-db07331bef52/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:03.5157192Z","hostname":"buskin","pid":1555,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/8fa4bc78-1026-41f0-b09d-c67224071376): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:03.515898105Z","hostname":"buskin","pid":1555,"component":"NexusNotifierTask","sled_id":"8fa4bc78-1026-41f0-b09d-c67224071376","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"Sled Agent Started","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:03.515908665Z","hostname":"buskin","pid":1555,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:138"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:03.515927555Z","hostname":"buskin","pid":1555,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:271","disks":"{}","underlay_network_info":"Some((fd00:1122:3344:101::2, Some(RackNetworkConfigV2 { rack_subnet: Ipv6Net { addr: fd00:1122:3344:100::, width: 56 }, infra_ip_first: 10.151.1.174, infra_ip_last: 10.151.1.174, ports: [PortConfigV2 { routes: [RouteConfig { destination: V4(Ipv4Net { addr: 0.0.0.0, width: 0 }), nexthop: 10.151.1.100, vlan_id: None }], addresses: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }], switch: Switch0, port: \"qsfp0\", uplink_port_speed: Speed40G, uplink_port_fec: None, bgp_peers: [], autoneg: false }], bgp: [], bfd: [] })))"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:03.515936715Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3665"} {"msg":"Connection closed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:03.515943066Z","hostname":"buskin","pid":1555,"remote_addr":"[fdb0:18c0:4d0d:9fb2::1]:57030","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:73"} {"msg":"Peer agent initialized","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:03.516031798Z","hostname":"buskin","pid":1555,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:108","peer":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Initialized sled agent","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:03.516037858Z","hostname":"buskin","pid":1555,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:232","target_sled":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:03.701137593Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:03.702160388Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"No ledger in /pool/int/6d2aa15e-618e-402e-b368-cf27fd8def68/config/rss-service-plan-v3.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:03.702310801Z","hostname":"buskin","pid":1555,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/75c788e1-45ef-4fcf-958d-db07331bef52/config/rss-service-plan-v3.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:03.702332372Z","hostname":"buskin","pid":1555,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:03.702364062Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:03.702386523Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:03.738555572Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:63451"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:03.738717046Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:03.738802678Z","hostname":"buskin","pid":1555,"uri":"/inventory","method":"GET","req_id":"045b0d14-d814-4535-a4eb-6b7364b324f0","remote_addr":"[fd00:1122:3344:101::1]:63451","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":146,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:03.770654904Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:49309"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:03.770790758Z","hostname":"buskin","pid":1555,"uri":"/sled-role","method":"GET","req_id":"9e90c4bc-0876-4e78-b043-be37c153c455","remote_addr":"[fd00:1122:3344:101::1]:49309","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":39,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:03.772025767Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Writing ledger to /pool/int/6d2aa15e-618e-402e-b368-cf27fd8def68/config/.rss-service-plan-v3.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:03.772067718Z","hostname":"buskin","pid":1555,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/75c788e1-45ef-4fcf-958d-db07331bef52/config/.rss-service-plan-v3.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:03.772459727Z","hostname":"buskin","pid":1555,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Service plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:03.772760724Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/plan/service.rs:832"} {"msg":"attempting to set up sled's storage: OmicronPhysicalDisksConfig { disks: [OmicronPhysicalDiskConfig { id: 489d3050-bf57-46d2-bf46-7cd957436e3d, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, pool_id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool) }, OmicronPhysicalDiskConfig { id: 4edc2975-bd7a-4aad-80f3-9409995355d1, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, pool_id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool) }, OmicronPhysicalDiskConfig { id: e2770ac5-7ae7-4600-bf20-0849191c7d7c, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, pool_id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool) }, OmicronPhysicalDiskConfig { id: 7b9e4bc9-bcfc-41a0-be73-09102123a06a, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, pool_id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool) }, OmicronPhysicalDiskConfig { id: d8e66d81-f03a-4b7f-94b4-e281f422c437, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, pool_id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool) }, OmicronPhysicalDiskConfig { id: 282f71da-0fb9-4a90-9caa-dd76e77730ca, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, pool_id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool) }, OmicronPhysicalDiskConfig { id: 0cd36a66-88d7-45b4-82f8-717f3515dbd9, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, pool_id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool) }, OmicronPhysicalDiskConfig { id: 6e68f196-dc3d-4e49-9aac-95ce41525cb3, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, pool_id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool) }, OmicronPhysicalDiskConfig { id: 3c83f3c1-50fb-429e-b2d4-58a398af121f, identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, pool_id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool) }], generation: Generation(1) }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:03.802819688Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:351"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:03.802971662Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:47856"} {"msg":"physical disks ensure","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:03.803210247Z","hostname":"buskin","pid":1555,"sled_id":"8fa4bc78-1026-41f0-b09d-c67224071376","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:824"} {"msg":"Received OmicronPhysicalDisksEnsure { config: OmicronPhysicalDisksConfig { generation: Generation(1), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, id: 489d3050-bf57-46d2-bf46-7cd957436e3d, pool_id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, id: 4edc2975-bd7a-4aad-80f3-9409995355d1, pool_id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, id: e2770ac5-7ae7-4600-bf20-0849191c7d7c, pool_id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, id: 7b9e4bc9-bcfc-41a0-be73-09102123a06a, pool_id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, id: d8e66d81-f03a-4b7f-94b4-e281f422c437, pool_id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, id: 282f71da-0fb9-4a90-9caa-dd76e77730ca, pool_id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, id: 0cd36a66-88d7-45b4-82f8-717f3515dbd9, pool_id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, id: 6e68f196-dc3d-4e49-9aac-95ce41525cb3, pool_id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, id: 3c83f3c1-50fb-429e-b2d4-58a398af121f, pool_id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool) }] }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:03.803247678Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"No ledger in /pool/int/6d2aa15e-618e-402e-b368-cf27fd8def68/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:03.803266549Z","hostname":"buskin","pid":1555,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/75c788e1-45ef-4fcf-958d-db07331bef52/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:03.803278349Z","hostname":"buskin","pid":1555,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No previously-stored 'requested disks', creating new ledger","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:03.803285809Z","hostname":"buskin","pid":1555,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:643"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:03.80332568Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:374"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:03.80333457Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:414","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:03.8033431Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:491"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:03.803350271Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:127","path":"/scratch/u2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:03.906606883Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","component":"MetricsManager","sled_id":"8fa4bc78-1026-41f0-b09d-c67224071376","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"709.360328ms"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:03.920280737Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:361"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:03.920290408Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367","id":"dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool)"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/8fa4bc78-1026-41f0-b09d-c67224071376): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-07-25T05:44:04.551855594Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"8fa4bc78-1026-41f0-b09d-c67224071376","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:04.738583348Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","component":"MetricsManager","sled_id":"8fa4bc78-1026-41f0-b09d-c67224071376","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"1.309974375s"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:04.938733271Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:05.12357073Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:05.123651202Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:05.123664342Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:05.123671452Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:05.123680303Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:05.123815806Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:05.123823886Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/8fa4bc78-1026-41f0-b09d-c67224071376): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-07-25T05:44:05.609419758Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"8fa4bc78-1026-41f0-b09d-c67224071376","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/8fa4bc78-1026-41f0-b09d-c67224071376): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:05.609450948Z","hostname":"buskin","pid":1555,"component":"NexusNotifierTask","sled_id":"8fa4bc78-1026-41f0-b09d-c67224071376","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"Setting up dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:05.799240905Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2476"} {"msg":"Setting up lldpd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:05.799476771Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2694"} {"msg":"Setting up MGS service","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:05.799482901Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2346"} {"msg":"Setting up wicketd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:05.799496621Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2393"} {"msg":"Setting up mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:05.799548802Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2775"} {"msg":"Setting up mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:05.799554222Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2817"} {"msg":"Setting up tfport service","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:05.801748314Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2619"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:05.802365009Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Setting up Simulated SP service","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:05.998953507Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2387"} {"msg":"Profile for oxz_switch:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:05.999126311Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_switch zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:05.999318305Z","hostname":"buskin","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"Automatically destroying dataset: oxp_dd1c81eb-5ddf-42bb-acfe-348d2d141fca/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:06.18438879Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:06.602840917Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","component":"MetricsManager","sled_id":"8fa4bc78-1026-41f0-b09d-c67224071376","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"2.264922361s"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 46.121999ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:06.781413908Z","hostname":"buskin","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/8fa4bc78-1026-41f0-b09d-c67224071376): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-07-25T05:44:06.783398195Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"8fa4bc78-1026-41f0-b09d-c67224071376","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 139.50719ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:07.30878182Z","hostname":"buskin","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 215.956821ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:07.914785491Z","hostname":"buskin","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/8fa4bc78-1026-41f0-b09d-c67224071376): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-07-25T05:44:07.916616744Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"8fa4bc78-1026-41f0-b09d-c67224071376","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/8fa4bc78-1026-41f0-b09d-c67224071376): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:07.916990503Z","hostname":"buskin","pid":1555,"component":"NexusNotifierTask","sled_id":"8fa4bc78-1026-41f0-b09d-c67224071376","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 474.054886ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:08.633866626Z","hostname":"buskin","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:08.870335261Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","component":"MetricsManager","sled_id":"8fa4bc78-1026-41f0-b09d-c67224071376","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"3.701114401s"} {"msg":"Looking for unencrypted datasets in oxp_dd1c81eb-5ddf-42bb-acfe-348d2d141fca","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:08.879620712Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/8fa4bc78-1026-41f0-b09d-c67224071376): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-07-25T05:44:09.335421555Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"8fa4bc78-1026-41f0-b09d-c67224071376","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:09.335926747Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:512","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:09.335939707Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:429","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:09.335947087Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:414","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:09.335953788Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:491"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:09.335968068Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:127","path":"/scratch/u2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 1.097160834s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:09.549221032Z","hostname":"buskin","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:09.559417664Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:361"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:09.559425594Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367","id":"59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool)"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/8fa4bc78-1026-41f0-b09d-c67224071376): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:10.007834612Z","hostname":"buskin","pid":1555,"component":"NexusNotifierTask","sled_id":"8fa4bc78-1026-41f0-b09d-c67224071376","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 942.720955ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:10.96577532Z","hostname":"buskin","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/8fa4bc78-1026-41f0-b09d-c67224071376): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-07-25T05:44:10.965823011Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"8fa4bc78-1026-41f0-b09d-c67224071376","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:10.975971031Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:11.214652609Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:11.21469381Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:11.21470116Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:11.21470985Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:11.214717281Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:11.214910345Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:11.214917545Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:12.289279017Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/8fa4bc78-1026-41f0-b09d-c67224071376): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-07-25T05:44:12.795958409Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"8fa4bc78-1026-41f0-b09d-c67224071376","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/8fa4bc78-1026-41f0-b09d-c67224071376): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:12.7960188Z","hostname":"buskin","pid":1555,"component":"NexusNotifierTask","sled_id":"8fa4bc78-1026-41f0-b09d-c67224071376","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:12.79602778Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","component":"MetricsManager","sled_id":"8fa4bc78-1026-41f0-b09d-c67224071376","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"9.144510494s"} {"msg":"Automatically destroying dataset: oxp_59d3cc5a-40b7-44b8-b2af-cfc21755b8bf/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:12.796038311Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Re-enabling running switch zone (new address)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:13.070197491Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3923","new":"[fd00:1122:3344:101::2, ::1]","old":"[::1]"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:13.070268602Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3939"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::2, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:13.070281453Z","hostname":"buskin","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:564"} {"msg":"Finished setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:13.998648448Z","hostname":"buskin","pid":1555,"component":"VmmReservoirManager","sled_id":"8fa4bc78-1026-41f0-b09d-c67224071376","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:279"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/8fa4bc78-1026-41f0-b09d-c67224071376): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-07-25T05:44:14.007324814Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"8fa4bc78-1026-41f0-b09d-c67224071376","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/8fa4bc78-1026-41f0-b09d-c67224071376): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:14.007665862Z","hostname":"buskin","pid":1555,"component":"NexusNotifierTask","sled_id":"8fa4bc78-1026-41f0-b09d-c67224071376","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"Looking for unencrypted datasets in oxp_59d3cc5a-40b7-44b8-b2af-cfc21755b8bf","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:14.086274768Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:14.094153905Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:512","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:14.094204737Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:429","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:14.094215827Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:414","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:14.094223497Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:491"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:14.094229787Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:127","path":"/scratch/u2_2.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_2.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:14.103696082Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:361"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:14.103735923Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367","id":"e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:14.194685003Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","zpool":"ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:14.201657168Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:14.201700689Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:14.201711479Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:14.20172651Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:14.20173801Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:14.201877014Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:14.201888204Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:14.236100716Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_e78b4f40-df20-4bcf-bd8a-b80afa109d70/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:14.243113983Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_e78b4f40-df20-4bcf-bd8a-b80afa109d70","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:14.363930312Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:14.372497995Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:512","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:14.372531766Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:429","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:14.372538996Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:414","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:14.372545926Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:491"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:14.372559887Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:127","path":"/scratch/u2_3.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_3.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:14.382241546Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:361"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:14.382276407Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367","id":"63a12260-6721-4f51-85d5-262aedc90628 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:14.480153652Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","zpool":"ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:14.488272134Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:14.488311195Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:14.488337386Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:14.48849242Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:14.48850243Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:14.488597052Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:14.488603852Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:14.524889804Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_63a12260-6721-4f51-85d5-262aedc90628/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:14.532678579Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_63a12260-6721-4f51-85d5-262aedc90628","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:14.65862378Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:14.666568738Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:512","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:14.666615459Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:429","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:14.66662587Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:414","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:14.66664167Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:491"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:14.66664752Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:127","path":"/scratch/u2_4.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_4.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:14.676683938Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:361"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:14.676725739Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367","id":"587239d7-a145-4bb7-b24a-aff014195deb (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:14.769501663Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","zpool":"ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:14.777558374Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:14.777594985Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:14.777600495Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:14.777636016Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:14.777645956Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:14.777785909Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:14.777795099Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:14.813378564Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_587239d7-a145-4bb7-b24a-aff014195deb/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:14.821384595Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_587239d7-a145-4bb7-b24a-aff014195deb","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:14.943809782Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:14.952030927Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:512","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:14.952066198Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:429","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:14.952072238Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:414","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:14.952077718Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:491"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:14.952081828Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:127","path":"/scratch/u2_5.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_5.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:14.962303971Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:361"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:14.962335122Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367","id":"50e63493-6b9c-42de-846b-797a312b5fbc (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:15.055726339Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","zpool":"ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:15.063463023Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:15.063509854Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:15.063516064Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:15.063551435Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:15.063564835Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:15.06376753Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:15.063779Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/8fa4bc78-1026-41f0-b09d-c67224071376): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-07-25T05:44:15.067753545Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"8fa4bc78-1026-41f0-b09d-c67224071376","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:15.101927246Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_50e63493-6b9c-42de-846b-797a312b5fbc/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:15.109694591Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_50e63493-6b9c-42de-846b-797a312b5fbc","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:15.23345203Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:15.241842509Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:512","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:15.24190209Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:429","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:15.24191553Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:414","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:15.241927841Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:491"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:15.241946601Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:127","path":"/scratch/u2_6.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_6.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:15.252149603Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:361"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:15.252183444Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367","id":"fd75587e-5d72-48df-a44e-8602b2d61469 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:15.349179847Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","zpool":"ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:15.357238449Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:15.35729382Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:15.357305531Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:15.357322851Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:15.357337381Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:15.357472275Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:15.357491785Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:15.392802794Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_fd75587e-5d72-48df-a44e-8602b2d61469/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:15.400285391Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_fd75587e-5d72-48df-a44e-8602b2d61469","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:15.519283097Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:15.527092372Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:512","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:15.527136404Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:429","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:15.527146224Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:414","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:15.527154264Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:491"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:15.527159514Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:127","path":"/scratch/u2_7.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_7.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:15.536620339Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:361"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:15.536654339Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367","id":"20d995c8-4908-4380-8438-c04a6e3adea2 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:15.626710008Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","zpool":"ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:15.634419801Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:15.634474172Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:15.634484383Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:15.634524003Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:15.634561824Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:15.634714458Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:15.634732819Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:15.669536595Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_20d995c8-4908-4380-8438-c04a6e3adea2/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:15.67733371Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Looking for unencrypted datasets in oxp_20d995c8-4908-4380-8438-c04a6e3adea2","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:15.802209396Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:15.810001281Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:512","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:15.810054362Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:429","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:15.810065502Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:414","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:15.810071652Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:491"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:15.810076752Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:127","path":"/scratch/u2_8.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_8.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:15.819795633Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:361"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:15.819839234Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:367","id":"e43689a1-378e-4be8-9661-3bc522647c6d (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:15.915886585Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:288","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","zpool":"ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:15.923747772Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:332","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:15.923818203Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:334","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:15.923835124Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:338","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:15.923847404Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:340","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:15.923859664Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:15.923968797Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:15.923984437Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:352"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:15.961842776Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_e43689a1-378e-4be8-9661-3bc522647c6d/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:15.974134807Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:395"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/8fa4bc78-1026-41f0-b09d-c67224071376): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:16.001769224Z","hostname":"buskin","pid":1555,"component":"NexusNotifierTask","sled_id":"8fa4bc78-1026-41f0-b09d-c67224071376","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/8fa4bc78-1026-41f0-b09d-c67224071376): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-07-25T05:44:16.070014974Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"8fa4bc78-1026-41f0-b09d-c67224071376","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"Looking for unencrypted datasets in oxp_e43689a1-378e-4be8-9661-3bc522647c6d","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:16.105892626Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:480"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:16.114691605Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:512","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:16.114753107Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:429","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Writing ledger to /pool/int/6d2aa15e-618e-402e-b368-cf27fd8def68/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:16.114792498Z","hostname":"buskin","pid":1555,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:16.115019533Z","hostname":"buskin","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 6d2aa15e-618e-402e-b368-cf27fd8def68 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 75c788e1-45ef-4fcf-958d-db07331bef52 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:16.115095085Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:16.115103685Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Writing ledger to /pool/int/75c788e1-45ef-4fcf-958d-db07331bef52/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:16.115108605Z","hostname":"buskin","pid":1555,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"physical disks ensure: Updated storage","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:16.11531637Z","hostname":"buskin","pid":1555,"sled_id":"8fa4bc78-1026-41f0-b09d-c67224071376","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:828"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:16.115338661Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"physical disks ensure: Propagating new generation of disks","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:16.115355751Z","hostname":"buskin","pid":1555,"sled_id":"8fa4bc78-1026-41f0-b09d-c67224071376","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:854","generation":"Generation(1)"} {"msg":"physical disks ensure: Updated storage monitor","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:16.115365411Z","hostname":"buskin","pid":1555,"sled_id":"8fa4bc78-1026-41f0-b09d-c67224071376","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:859"} {"msg":"physical disks ensure: Updated zone bundler","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:16.115370781Z","hostname":"buskin","pid":1555,"sled_id":"8fa4bc78-1026-41f0-b09d-c67224071376","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:864"} {"msg":"use_only_these_disks: Processing new request","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:16.115377092Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"8fa4bc78-1026-41f0-b09d-c67224071376","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:116","gen":"Generation(1)"} {"msg":"physical disks ensure: Updated probes","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:16.115381782Z","hostname":"buskin","pid":1555,"sled_id":"8fa4bc78-1026-41f0-b09d-c67224071376","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:870"} {"msg":"use_only_these_disks: Processing new request","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:16.115408832Z","hostname":"buskin","pid":1555,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:822","gen":"Generation(1)"} {"msg":"physical disks ensure: Updated instances","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:16.115414753Z","hostname":"buskin","pid":1555,"sled_id":"8fa4bc78-1026-41f0-b09d-c67224071376","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:875"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:16.115484444Z","hostname":"buskin","pid":1555,"uri":"/omicron-physical-disks","method":"PUT","req_id":"95b708ff-48ee-45ce-bc65-f5ab677dafa3","remote_addr":"[fd00:1122:3344:101::1]:47856","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":12312412,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(2), zones: [OmicronZoneConfig { id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:16.146301706Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:434"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:16.146429349Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:42574"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:16.146753287Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:16.146761597Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:16.198872344Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:16.198908395Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:16.251519405Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:16.251562726Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:16.303032068Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"No ledger in /pool/int/6d2aa15e-618e-402e-b368-cf27fd8def68/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:16.303069558Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/75c788e1-45ef-4fcf-958d-db07331bef52/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:16.303080889Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:16.30314449Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:16.348139279Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:16.34818513Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:16.3481896Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_0ee9b880-7690-4963-bc6b-3dbf4d39892d","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:16.403965865Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_aa38c6ce-70ac-4e2b-a383-8201d1e086e8","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:16.407865097Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_23e793cc-4c0e-4aca-92a8-2b30f6dafa5a","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:16.411920834Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_internal_dns_0ee9b880-7690-4963-bc6b-3dbf4d39892d","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:16.423468008Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_internal_dns_aa38c6ce-70ac-4e2b-a383-8201d1e086e8","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:16.427615876Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_internal_dns_23e793cc-4c0e-4aca-92a8-2b30f6dafa5a","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:16.432341478Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/8fa4bc78-1026-41f0-b09d-c67224071376): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-07-25T05:44:17.07252172Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"8fa4bc78-1026-41f0-b09d-c67224071376","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"Profile for oxz_internal_dns_0ee9b880-7690-4963-bc6b-3dbf4d39892d:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:17.933762281Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:17.933885404Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:2::, width: 64 }"} {"msg":"Booting oxz_internal_dns_0ee9b880-7690-4963-bc6b-3dbf4d39892d zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:17.934062729Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_0ee9b880-7690-4963-bc6b-3dbf4d39892d","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"Profile for oxz_internal_dns_aa38c6ce-70ac-4e2b-a383-8201d1e086e8:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:18.420906719Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_internal_dns_23e793cc-4c0e-4aca-92a8-2b30f6dafa5a:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:18.898571792Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:18.898636393Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:1::, width: 64 }"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:18.898650344Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:3::, width: 64 }"} {"msg":"wait for service Some(\"oxz_internal_dns_0ee9b880-7690-4963-bc6b-3dbf4d39892d\") failed: Property not found. retry in 26.397452ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:18.906655984Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_0ee9b880-7690-4963-bc6b-3dbf4d39892d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Booting oxz_internal_dns_aa38c6ce-70ac-4e2b-a383-8201d1e086e8 zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:18.906699645Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_aa38c6ce-70ac-4e2b-a383-8201d1e086e8","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"Booting oxz_internal_dns_23e793cc-4c0e-4aca-92a8-2b30f6dafa5a zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:18.91068715Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_23e793cc-4c0e-4aca-92a8-2b30f6dafa5a","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"wait for service Some(\"oxz_internal_dns_0ee9b880-7690-4963-bc6b-3dbf4d39892d\") failed: Property not found. retry in 143.100954ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:18.943206482Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_0ee9b880-7690-4963-bc6b-3dbf4d39892d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_aa38c6ce-70ac-4e2b-a383-8201d1e086e8\") failed: Property not found. retry in 46.303751ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:19.040345077Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_aa38c6ce-70ac-4e2b-a383-8201d1e086e8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_23e793cc-4c0e-4aca-92a8-2b30f6dafa5a\") failed: Property not found. retry in 29.466021ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:19.046650317Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_23e793cc-4c0e-4aca-92a8-2b30f6dafa5a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_23e793cc-4c0e-4aca-92a8-2b30f6dafa5a\") failed: Property not found. retry in 99.926536ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:19.082889608Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_23e793cc-4c0e-4aca-92a8-2b30f6dafa5a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_0ee9b880-7690-4963-bc6b-3dbf4d39892d\") failed: Property not found. retry in 170.237464ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:19.094926124Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_0ee9b880-7690-4963-bc6b-3dbf4d39892d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_aa38c6ce-70ac-4e2b-a383-8201d1e086e8\") failed: Property not found. retry in 80.909948ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:19.102427372Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_aa38c6ce-70ac-4e2b-a383-8201d1e086e8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_aa38c6ce-70ac-4e2b-a383-8201d1e086e8\") failed: Property not found. retry in 133.583981ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:19.24805816Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_aa38c6ce-70ac-4e2b-a383-8201d1e086e8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_23e793cc-4c0e-4aca-92a8-2b30f6dafa5a\") failed: Property not found. retry in 259.948447ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:19.314247532Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_23e793cc-4c0e-4aca-92a8-2b30f6dafa5a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_0ee9b880-7690-4963-bc6b-3dbf4d39892d\") failed: Property not found. retry in 458.496542ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:19.331799069Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_0ee9b880-7690-4963-bc6b-3dbf4d39892d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_aa38c6ce-70ac-4e2b-a383-8201d1e086e8\") failed: Property not found. retry in 536.888057ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:19.431512237Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_aa38c6ce-70ac-4e2b-a383-8201d1e086e8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_23e793cc-4c0e-4aca-92a8-2b30f6dafa5a\") failed: Property not found. retry in 525.481687ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:19.586690951Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_23e793cc-4c0e-4aca-92a8-2b30f6dafa5a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_0ee9b880-7690-4963-bc6b-3dbf4d39892d\") failed: Property not found. retry in 1.040577231s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:19.79887025Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_0ee9b880-7690-4963-bc6b-3dbf4d39892d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_aa38c6ce-70ac-4e2b-a383-8201d1e086e8\") failed: Property not found. retry in 638.40132ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:19.977621284Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_aa38c6ce-70ac-4e2b-a383-8201d1e086e8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_23e793cc-4c0e-4aca-92a8-2b30f6dafa5a\") failed: Property not found. retry in 553.274192ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:20.123755684Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_23e793cc-4c0e-4aca-92a8-2b30f6dafa5a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_aa38c6ce-70ac-4e2b-a383-8201d1e086e8\") failed: Property not found. retry in 1.173917497s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:20.63144045Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_aa38c6ce-70ac-4e2b-a383-8201d1e086e8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_23e793cc-4c0e-4aca-92a8-2b30f6dafa5a\") failed: Property not found. retry in 1.2476454s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:20.686692172Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_23e793cc-4c0e-4aca-92a8-2b30f6dafa5a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_0ee9b880-7690-4963-bc6b-3dbf4d39892d\") failed: Property not found. retry in 754.060743ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:20.84871557Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_0ee9b880-7690-4963-bc6b-3dbf4d39892d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_0ee9b880-7690-4963-bc6b-3dbf4d39892d\") failed: Property not found. retry in 1.370269374s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:21.612153978Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_0ee9b880-7690-4963-bc6b-3dbf4d39892d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_aa38c6ce-70ac-4e2b-a383-8201d1e086e8\") failed: Property not found. retry in 514.98304ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:21.813467239Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_aa38c6ce-70ac-4e2b-a383-8201d1e086e8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_23e793cc-4c0e-4aca-92a8-2b30f6dafa5a\") failed: Property not found. retry in 1.288095497s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:21.942566114Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_23e793cc-4c0e-4aca-92a8-2b30f6dafa5a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_aa38c6ce-70ac-4e2b-a383-8201d1e086e8\") failed: Property not found. retry in 1.191238046s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:22.337094942Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_aa38c6ce-70ac-4e2b-a383-8201d1e086e8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:22.488026506Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3947"} {"msg":"Ensuring there is a default route","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:22.488087758Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3957","gateway":"fd00:1122:3344:101::1"} {"msg":"configuring dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:22.493886546Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:4036"} {"msg":"refreshed dendrite service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:22.623574185Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:4084"} {"msg":"configuring lldp service","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:22.623629637Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:4111"} {"msg":"refreshed lldpd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:22.662060069Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:4124"} {"msg":"configuring MGS service","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:22.66209921Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3988"} {"msg":"refreshed MGS service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:22.715859207Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:4030"} {"msg":"configuring wicketd","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:22.715898508Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:4091","rack_subnet":"fd00:1122:3344::"} {"msg":"refreshed wicketd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:22.736092147Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:4102"} {"msg":"configuring mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:22.736123198Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:4144"} {"msg":"refreshed mgd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:22.829768892Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:4176"} {"msg":"configuring mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:22.829813573Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:4182"} {"msg":"refreshed mg-ddm service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:22.947553569Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:4223"} {"msg":"Determining physical location of our switch zone at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:22.94760493Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:364"} {"msg":"Failed to get switch ID from MGS (retrying in 38.179016ms)","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:22.979917087Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:383","error":"Communication Error: error sending request for url (http://[fd00:1122:3344:101::2]:12225/local/switch-id): error trying to connect: tcp connect error: Connection refused (os error 146)"} {"msg":"wait for service Some(\"oxz_internal_dns_0ee9b880-7690-4963-bc6b-3dbf4d39892d\") failed: Property not found. retry in 531.090446ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:22.992001753Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_0ee9b880-7690-4963-bc6b-3dbf4d39892d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Failed to get switch ID from MGS (retrying in 143.80073ms)","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:23.019728522Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:383","error":"Communication Error: error sending request for url (http://[fd00:1122:3344:101::2]:12225/local/switch-id): error trying to connect: tcp connect error: Connection refused (os error 146)"} {"msg":"Initializing 1 Uplinks on Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:23.165117224Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:412"} {"msg":"Building Port Configuration","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:23.196634673Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:651"} {"msg":"Checking dendrite uptime","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:23.196720385Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:689"} {"msg":"Dendrite online","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:23.197628187Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:692","uptime":"0"} {"msg":"Configuring default uplink on switch","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:23.197644427Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:434","config":"PortSettings {\n links: {\n \"0\": LinkSettings {\n addrs: [\n 10.151.1.174,\n ],\n params: LinkCreate {\n autoneg: false,\n fec: None,\n kr: false,\n lane: Some(\n LinkId(\n 0,\n ),\n ),\n speed: Speed40G,\n },\n },\n },\n}"} {"msg":"wait for service Some(\"oxz_internal_dns_23e793cc-4c0e-4aca-92a8-2b30f6dafa5a\") failed: Property not found. retry in 1.0106047s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:23.24200469Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_23e793cc-4c0e-4aca-92a8-2b30f6dafa5a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:23.242861711Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3824"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:23.26472157Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3835"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:23.284614122Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:23.284682674Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:601","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:23.284692884Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:622","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:23.284701484Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:601","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:23.284708964Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:622","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:23.284715624Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:601","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:23.284731325Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:622","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:23.284738255Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:601","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:23.284745055Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:622","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:23.284753625Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:601","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:23.284760336Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:622","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:23.284767236Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:601","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:23.284773806Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:622","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:23.284780606Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:601","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:23.284788366Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:622","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:23.284795017Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:601","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:23.284800226Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:622","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:23.284804777Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:601","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:23.284809457Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:622","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:23.284823187Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:601","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:23.284828237Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:622","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:23.284832967Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:601","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:23.284837627Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:622","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"wait for service Some(\"oxz_internal_dns_aa38c6ce-70ac-4e2b-a383-8201d1e086e8\") failed: Property not found. retry in 1.370220737s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:23.545470937Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_aa38c6ce-70ac-4e2b-a383-8201d1e086e8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:24.935095205Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_internal_dns_0ee9b880-7690-4963-bc6b-3dbf4d39892d"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:24.935154896Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_internal_dns_aa38c6ce-70ac-4e2b-a383-8201d1e086e8"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:24.935163487Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_internal_dns_23e793cc-4c0e-4aca-92a8-2b30f6dafa5a"} {"msg":"Writing ledger to /pool/int/6d2aa15e-618e-402e-b368-cf27fd8def68/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:24.935176617Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/75c788e1-45ef-4fcf-958d-db07331bef52/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:24.935486995Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:24.93573517Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"bcce3176-f156-4ea8-b45b-753af54b39fa","remote_addr":"[fd00:1122:3344:101::1]:42574","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":8789225,"response_code":"204"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:24.937340048Z","hostname":"buskin","pid":1555,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:552"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:24.982777746Z","hostname":"buskin","pid":1555,"dns_config_addr":"[fd00:1122:3344:2::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:552"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:25.029298381Z","hostname":"buskin","pid":1555,"dns_config_addr":"[fd00:1122:3344:3::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:552"} {"msg":"Configured all DNS servers","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:25.073483311Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:588"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:25.073495041Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:258"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:25.103322299Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:275","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:25.104083697Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:300"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:25.104106888Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:192","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(3), zones: [OmicronZoneConfig { id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3f311cd5-ee38-4707-ab8a-9331491f8e7c, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6389ba5c-9d50-4fe4-b1a2-8be3c6861cb0, kind: Service { id: 3f311cd5-ee38-4707-ab8a-9331491f8e7c }, name: Name(\"ntp-3f311cd5-ee38-4707-ab8a-9331491f8e7c\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 235])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:25.132582264Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:434"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:25.132690686Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:59412"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:25.132883351Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:25.132894401Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:25.152565218Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:25.15261489Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:25.17074454Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:25.170788211Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:25.188594744Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/6d2aa15e-618e-402e-b368-cf27fd8def68/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:25.188689056Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/75c788e1-45ef-4fcf-958d-db07331bef52/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:25.188872951Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:25.188973643Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:25.201231624Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:258"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:25.233352367Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:275","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:25.234017753Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:300"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:25.234027313Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:192","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:25.263676107Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:426","port":"Port { inner: PortInner(PortData { name: \"opte0\", ip: 172.30.3.5, mac: MacAddr6([168, 64, 37, 255, 246, 235]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.3.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:25.263698857Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:1207","zone_type":"ntp"} {"msg":"no nat entry found for: 10.151.1.169","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:25.264011035Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.169","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:25.264022965Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"creation of nat entry successful for: 10.151.1.169","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:25.264256131Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:25.264669971Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Configuring new Omicron zone: oxz_ntp_3f311cd5-ee38-4707-ab8a-9331491f8e7c","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:25.288787763Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_ntp_3f311cd5-ee38-4707-ab8a-9331491f8e7c","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:25.306697418Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_ntp_3f311cd5-ee38-4707-ab8a-9331491f8e7c:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:25.917547394Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_ntp_3f311cd5-ee38-4707-ab8a-9331491f8e7c zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:25.917627776Z","hostname":"buskin","pid":1555,"zone":"oxz_ntp_3f311cd5-ee38-4707-ab8a-9331491f8e7c","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"wait for service Some(\"oxz_ntp_3f311cd5-ee38-4707-ab8a-9331491f8e7c\") failed: Property not found. retry in 51.086364ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:26.051906964Z","hostname":"buskin","pid":1555,"zone":"oxz_ntp_3f311cd5-ee38-4707-ab8a-9331491f8e7c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_3f311cd5-ee38-4707-ab8a-9331491f8e7c\") failed: Property not found. retry in 134.138304ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:26.111728075Z","hostname":"buskin","pid":1555,"zone":"oxz_ntp_3f311cd5-ee38-4707-ab8a-9331491f8e7c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_3f311cd5-ee38-4707-ab8a-9331491f8e7c\") failed: Property not found. retry in 232.663274ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:26.288238877Z","hostname":"buskin","pid":1555,"zone":"oxz_ntp_3f311cd5-ee38-4707-ab8a-9331491f8e7c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_3f311cd5-ee38-4707-ab8a-9331491f8e7c\") failed: Property not found. retry in 444.971337ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:26.530063869Z","hostname":"buskin","pid":1555,"zone":"oxz_ntp_3f311cd5-ee38-4707-ab8a-9331491f8e7c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_3f311cd5-ee38-4707-ab8a-9331491f8e7c\") failed: Property not found. retry in 504.289589ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:26.987199154Z","hostname":"buskin","pid":1555,"zone":"oxz_ntp_3f311cd5-ee38-4707-ab8a-9331491f8e7c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_3f311cd5-ee38-4707-ab8a-9331491f8e7c\") failed: Property not found. retry in 930.850343ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:27.500677747Z","hostname":"buskin","pid":1555,"zone":"oxz_ntp_3f311cd5-ee38-4707-ab8a-9331491f8e7c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_3f311cd5-ee38-4707-ab8a-9331491f8e7c\") failed: Property not found. retry in 1.116323721s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:28.440459833Z","hostname":"buskin","pid":1555,"zone":"oxz_ntp_3f311cd5-ee38-4707-ab8a-9331491f8e7c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_3f311cd5-ee38-4707-ab8a-9331491f8e7c\") failed: Property not found. retry in 625.92626ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:29.567332192Z","hostname":"buskin","pid":1555,"zone":"oxz_ntp_3f311cd5-ee38-4707-ab8a-9331491f8e7c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_3f311cd5-ee38-4707-ab8a-9331491f8e7c\") failed: Property not found. retry in 1.435089965s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:30.202278629Z","hostname":"buskin","pid":1555,"zone":"oxz_ntp_3f311cd5-ee38-4707-ab8a-9331491f8e7c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:31.656115582Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_ntp_3f311cd5-ee38-4707-ab8a-9331491f8e7c"} {"msg":"Writing ledger to /pool/int/6d2aa15e-618e-402e-b368-cf27fd8def68/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:31.656174694Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/75c788e1-45ef-4fcf-958d-db07331bef52/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:31.656414889Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:31.656654315Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"b49e5d71-dd16-41d5-aa18-e48da7660345","remote_addr":"[fd00:1122:3344:101::1]:59412","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":6523934,"response_code":"204"} {"msg":"Waiting for rack time synchronization","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:31.658656252Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:629"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:31.688109252Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:609"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:31.688253546Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:61618"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_3f311cd5-ee38-4707-ab8a-9331491f8e7c': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:31.69726592Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3651"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:31.697487675Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"280a9a62-36e7-4a2e-9b2f-f7f94178266c","remote_addr":"[fd00:1122:3344:101::1]:61618","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":9051,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:31.700319462Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:659","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:31.83499876Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:609"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:31.835150724Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:32781"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_3f311cd5-ee38-4707-ab8a-9331491f8e7c': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:35.843065366Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3651"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:35.843192909Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"ec20e060-bffc-4081-b2dd-f18ed7baf527","remote_addr":"[fd00:1122:3344:101::1]:32781","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":4007957,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:35.847074221Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:659","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:35.986983703Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:609"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:35.987091725Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:61377"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_3f311cd5-ee38-4707-ab8a-9331491f8e7c': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:35.995589467Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3651"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:35.995700699Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"c58f9cc1-da75-4c16-b432-c03f8418085d","remote_addr":"[fd00:1122:3344:101::1]:61377","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":8522,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:35.997500732Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:659","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:36.155015783Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:609"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:36.155142596Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:53099"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:36.163079764Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"22f9368b-0178-4e51-b531-b29fe47ec928","remote_addr":"[fd00:1122:3344:101::1]:53099","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":7850,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:36.165494562Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:637"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:36.165505552Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:659","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:36.312899032Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:609"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:36.313009885Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:49518"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:36.319795726Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"faa31035-3896-4766-9752-dee84fc39350","remote_addr":"[fd00:1122:3344:101::1]:49518","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":6708,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:36.321702871Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:637"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:36.321716791Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:659","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:36.655308493Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:609"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:36.655401035Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:33858"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:36.662635997Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"5c5585e9-50c1-4f0d-84ac-682342d143fc","remote_addr":"[fd00:1122:3344:101::1]:33858","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":7168,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:36.66444089Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:637"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:36.66444664Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:659","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:36.836511316Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:609"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:36.836600568Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:44098"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:36.843412Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"bb08cafb-3ca4-4d51-9b39-13d2c724e61c","remote_addr":"[fd00:1122:3344:101::1]:44098","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":6747,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:36.845257853Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:637"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:36.845265264Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:659","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:37.049712388Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:609"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:37.049818581Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:38676"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:37.057122794Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"fb06a834-7719-46db-80ed-4929d9180c1e","remote_addr":"[fd00:1122:3344:101::1]:38676","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":7235,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:37.059131702Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:637"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:37.059139482Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:659","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:37.30034638Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:609"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:37.300451172Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:50734"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:37.308650637Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"564be9a2-38c9-4289-a6bd-0870346b00f1","remote_addr":"[fd00:1122:3344:101::1]:50734","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":8131,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: false, ref_id: 0, ip_addr: ::, stratum: 0, ref_time: 0.0, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:37.310734146Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:637"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:37.310741186Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:659","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:37.010382827Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:609"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:37.010470699Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:44710"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:37.030102685Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"be8b6594-e6a8-4076-8ec7-202a6d892048","remote_addr":"[fd00:1122:3344:101::1]:44710","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":19537,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: true, ref_id: 2728380539, ip_addr: 162.159.200.123, stratum: 4, ref_time: 1721886277.170336, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:37.032163354Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:637"} {"msg":"Finished setting up Internal DNS and NTP","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:37.032180114Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1227"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3f311cd5-ee38-4707-ab8a-9331491f8e7c, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6389ba5c-9d50-4fe4-b1a2-8be3c6861cb0, kind: Service { id: 3f311cd5-ee38-4707-ab8a-9331491f8e7c }, name: Name(\"ntp-3f311cd5-ee38-4707-ab8a-9331491f8e7c\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 235])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4f6a29ee-ca23-4fa8-be69-497060121028, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } } } }, OmicronZoneConfig { id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } } } }, OmicronZoneConfig { id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:37.062102425Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:434"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:37.062187157Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:33707"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:37.062373931Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:37.062384172Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:37.081236269Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:37.0812792Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:37.098698744Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:37.098738785Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4f6a29ee-ca23-4fa8-be69-497060121028, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:37.117713816Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:37.117758326Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4f6a29ee-ca23-4fa8-be69-497060121028, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:37.169382063Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:37.169431174Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:37.219545134Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:37.219591925Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:37.269743436Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:37.269795847Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:37.323193225Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:37.323242406Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:37.375721552Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/6d2aa15e-618e-402e-b368-cf27fd8def68/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:37.375774943Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/75c788e1-45ef-4fcf-958d-db07331bef52/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:37.375951557Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:37.37605696Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:37.457018122Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:37.457078734Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:37.457086174Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:37.457094674Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:37.457099874Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_4f6a29ee-ca23-4fa8-be69-497060121028","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:37.545837511Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_3b0743b7-2816-4de6-a40a-d863e46d23f8","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:37.549915559Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_d835fd87-0881-47dd-8e5b-ebb01fd7b0ce","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:37.554058517Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_4553d8e1-a7ca-4d20-b1a4-394640a4ad4f","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:37.558350379Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_33444b97-3e72-47f7-a319-cbd1ca92b6c8","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:37.562742343Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_cockroachdb_4f6a29ee-ca23-4fa8-be69-497060121028","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:37.567385943Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_3b0743b7-2816-4de6-a40a-d863e46d23f8","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:37.57186084Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_d835fd87-0881-47dd-8e5b-ebb01fd7b0ce","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:37.576281594Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_4553d8e1-a7ca-4d20-b1a4-394640a4ad4f","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:37.580655598Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_33444b97-3e72-47f7-a319-cbd1ca92b6c8","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:37.585177136Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_cockroachdb_4f6a29ee-ca23-4fa8-be69-497060121028:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:40.181191167Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_4f6a29ee-ca23-4fa8-be69-497060121028 zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:40.181324596Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_4f6a29ee-ca23-4fa8-be69-497060121028","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"Profile for oxz_cockroachdb_d835fd87-0881-47dd-8e5b-ebb01fd7b0ce:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:40.211484022Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_d835fd87-0881-47dd-8e5b-ebb01fd7b0ce zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:40.21159675Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_d835fd87-0881-47dd-8e5b-ebb01fd7b0ce","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"Profile for oxz_cockroachdb_3b0743b7-2816-4de6-a40a-d863e46d23f8:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:40.217714959Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_3b0743b7-2816-4de6-a40a-d863e46d23f8 zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:40.217937331Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_3b0743b7-2816-4de6-a40a-d863e46d23f8","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"Profile for oxz_cockroachdb_4553d8e1-a7ca-4d20-b1a4-394640a4ad4f:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:40.224875644Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_4553d8e1-a7ca-4d20-b1a4-394640a4ad4f zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:40.225007169Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_4553d8e1-a7ca-4d20-b1a4-394640a4ad4f","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"Profile for oxz_cockroachdb_33444b97-3e72-47f7-a319-cbd1ca92b6c8:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:40.316195423Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_33444b97-3e72-47f7-a319-cbd1ca92b6c8 zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:40.316326479Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_33444b97-3e72-47f7-a319-cbd1ca92b6c8","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"wait for service Some(\"oxz_cockroachdb_4f6a29ee-ca23-4fa8-be69-497060121028\") failed: Property not found. retry in 57.779835ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:40.329789726Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_4f6a29ee-ca23-4fa8-be69-497060121028","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d835fd87-0881-47dd-8e5b-ebb01fd7b0ce\") failed: Property not found. retry in 26.577257ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:40.35148839Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_d835fd87-0881-47dd-8e5b-ebb01fd7b0ce","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3b0743b7-2816-4de6-a40a-d863e46d23f8\") failed: Property not found. retry in 26.907565ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:40.359048575Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_3b0743b7-2816-4de6-a40a-d863e46d23f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4553d8e1-a7ca-4d20-b1a4-394640a4ad4f\") failed: Property not found. retry in 65.642048ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:40.365991283Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_4553d8e1-a7ca-4d20-b1a4-394640a4ad4f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d835fd87-0881-47dd-8e5b-ebb01fd7b0ce\") failed: Property not found. retry in 82.981268ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:40.38663069Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_d835fd87-0881-47dd-8e5b-ebb01fd7b0ce","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3b0743b7-2816-4de6-a40a-d863e46d23f8\") failed: Property not found. retry in 56.175315ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:40.407938592Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_3b0743b7-2816-4de6-a40a-d863e46d23f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4f6a29ee-ca23-4fa8-be69-497060121028\") failed: Property not found. retry in 53.594397ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:40.442739972Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_4f6a29ee-ca23-4fa8-be69-497060121028","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4553d8e1-a7ca-4d20-b1a4-394640a4ad4f\") failed: Property not found. retry in 58.188569ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:40.595144457Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_4553d8e1-a7ca-4d20-b1a4-394640a4ad4f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:41.095548268Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","component":"MetricsManager","sled_id":"8fa4bc78-1026-41f0-b09d-c67224071376","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"218.440209ms"} {"msg":"wait for service Some(\"oxz_cockroachdb_4f6a29ee-ca23-4fa8-be69-497060121028\") failed: Property not found. retry in 228.160614ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:41.382641249Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_4f6a29ee-ca23-4fa8-be69-497060121028","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3b0743b7-2816-4de6-a40a-d863e46d23f8\") failed: Property not found. retry in 214.745196ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:41.432645585Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_3b0743b7-2816-4de6-a40a-d863e46d23f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d835fd87-0881-47dd-8e5b-ebb01fd7b0ce\") failed: Property not found. retry in 238.331772ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:41.609905974Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_d835fd87-0881-47dd-8e5b-ebb01fd7b0ce","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4553d8e1-a7ca-4d20-b1a4-394640a4ad4f\") failed: Property not found. retry in 218.315032ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:41.719590448Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_4553d8e1-a7ca-4d20-b1a4-394640a4ad4f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_33444b97-3e72-47f7-a319-cbd1ca92b6c8\") failed: Property not found. retry in 61.433144ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:42.086675558Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_33444b97-3e72-47f7-a319-cbd1ca92b6c8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4f6a29ee-ca23-4fa8-be69-497060121028\") failed: Property not found. retry in 303.237755ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:42.307579737Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_4f6a29ee-ca23-4fa8-be69-497060121028","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3b0743b7-2816-4de6-a40a-d863e46d23f8\") failed: Property not found. retry in 521.947244ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:42.390253911Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_3b0743b7-2816-4de6-a40a-d863e46d23f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d835fd87-0881-47dd-8e5b-ebb01fd7b0ce\") failed: Property not found. retry in 452.137631ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:42.48662695Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_d835fd87-0881-47dd-8e5b-ebb01fd7b0ce","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4553d8e1-a7ca-4d20-b1a4-394640a4ad4f\") failed: Property not found. retry in 222.932796ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:42.55692846Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_4553d8e1-a7ca-4d20-b1a4-394640a4ad4f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_33444b97-3e72-47f7-a319-cbd1ca92b6c8\") failed: Property not found. retry in 68.159842ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:42.588049464Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_33444b97-3e72-47f7-a319-cbd1ca92b6c8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4f6a29ee-ca23-4fa8-be69-497060121028\") failed: Property not found. retry in 413.316286ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:42.629825654Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_4f6a29ee-ca23-4fa8-be69-497060121028","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_33444b97-3e72-47f7-a319-cbd1ca92b6c8\") failed: Property not found. retry in 173.878027ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:42.676325329Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_33444b97-3e72-47f7-a319-cbd1ca92b6c8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4553d8e1-a7ca-4d20-b1a4-394640a4ad4f\") failed: Property not found. retry in 993.084116ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:42.801616306Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_4553d8e1-a7ca-4d20-b1a4-394640a4ad4f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_33444b97-3e72-47f7-a319-cbd1ca92b6c8\") failed: Property not found. retry in 570.249322ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:42.863488434Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_33444b97-3e72-47f7-a319-cbd1ca92b6c8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3b0743b7-2816-4de6-a40a-d863e46d23f8\") failed: Property not found. retry in 474.450711ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:42.924504913Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_3b0743b7-2816-4de6-a40a-d863e46d23f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d835fd87-0881-47dd-8e5b-ebb01fd7b0ce\") failed: Property not found. retry in 840.976815ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:42.951087216Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_d835fd87-0881-47dd-8e5b-ebb01fd7b0ce","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4f6a29ee-ca23-4fa8-be69-497060121028\") failed: Property not found. retry in 935.132107ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:43.056488665Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_4f6a29ee-ca23-4fa8-be69-497060121028","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3b0743b7-2816-4de6-a40a-d863e46d23f8\") failed: Property not found. retry in 1.255322408s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:43.412697027Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_3b0743b7-2816-4de6-a40a-d863e46d23f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_33444b97-3e72-47f7-a319-cbd1ca92b6c8\") failed: Property not found. retry in 413.254558ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:43.447271334Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_33444b97-3e72-47f7-a319-cbd1ca92b6c8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d835fd87-0881-47dd-8e5b-ebb01fd7b0ce\") failed: Property not found. retry in 1.142773802s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:43.804467903Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_d835fd87-0881-47dd-8e5b-ebb01fd7b0ce","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4553d8e1-a7ca-4d20-b1a4-394640a4ad4f\") failed: Property not found. retry in 713.47099ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:43.812998513Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_4553d8e1-a7ca-4d20-b1a4-394640a4ad4f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_33444b97-3e72-47f7-a319-cbd1ca92b6c8\") failed: Property not found. retry in 854.592819ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:43.869690391Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_33444b97-3e72-47f7-a319-cbd1ca92b6c8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4f6a29ee-ca23-4fa8-be69-497060121028\") failed: Property not found. retry in 867.538176ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:44.004079349Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_4f6a29ee-ca23-4fa8-be69-497060121028","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4553d8e1-a7ca-4d20-b1a4-394640a4ad4f\") failed: Property not found. retry in 751.17122ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:44.537488305Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_4553d8e1-a7ca-4d20-b1a4-394640a4ad4f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3b0743b7-2816-4de6-a40a-d863e46d23f8\") failed: Property not found. retry in 544.783176ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:44.680213332Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_3b0743b7-2816-4de6-a40a-d863e46d23f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_33444b97-3e72-47f7-a319-cbd1ca92b6c8\") failed: Property not found. retry in 811.377988ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:44.735372813Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_33444b97-3e72-47f7-a319-cbd1ca92b6c8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4f6a29ee-ca23-4fa8-be69-497060121028\") failed: Property not found. retry in 1.371962857s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:44.882630314Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_4f6a29ee-ca23-4fa8-be69-497060121028","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d835fd87-0881-47dd-8e5b-ebb01fd7b0ce\") failed: Property not found. retry in 660.375253ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:44.959475193Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_d835fd87-0881-47dd-8e5b-ebb01fd7b0ce","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3b0743b7-2816-4de6-a40a-d863e46d23f8\") failed: Property not found. retry in 754.880337ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:45.237389886Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_3b0743b7-2816-4de6-a40a-d863e46d23f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4553d8e1-a7ca-4d20-b1a4-394640a4ad4f\") failed: Property not found. retry in 669.450852ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:45.302299506Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_4553d8e1-a7ca-4d20-b1a4-394640a4ad4f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_33444b97-3e72-47f7-a319-cbd1ca92b6c8\") failed: Property not found. retry in 1.382036708s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:45.559669295Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_33444b97-3e72-47f7-a319-cbd1ca92b6c8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d835fd87-0881-47dd-8e5b-ebb01fd7b0ce\") failed: Property not found. retry in 822.488873ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:45.632409069Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_d835fd87-0881-47dd-8e5b-ebb01fd7b0ce","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4553d8e1-a7ca-4d20-b1a4-394640a4ad4f\") failed: Property not found. retry in 955.606974ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:45.983768568Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_4553d8e1-a7ca-4d20-b1a4-394640a4ad4f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3b0743b7-2816-4de6-a40a-d863e46d23f8\") failed: Property not found. retry in 974.520096ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:46.002830345Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_3b0743b7-2816-4de6-a40a-d863e46d23f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4f6a29ee-ca23-4fa8-be69-497060121028\") failed: Property not found. retry in 1.327877443s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:46.268212157Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_4f6a29ee-ca23-4fa8-be69-497060121028","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d835fd87-0881-47dd-8e5b-ebb01fd7b0ce\") failed: Property not found. retry in 1.069057515s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:46.46799214Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_d835fd87-0881-47dd-8e5b-ebb01fd7b0ce","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4553d8e1-a7ca-4d20-b1a4-394640a4ad4f\") failed: Property not found. retry in 869.566296ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:46.952480073Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_4553d8e1-a7ca-4d20-b1a4-394640a4ad4f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_33444b97-3e72-47f7-a319-cbd1ca92b6c8\") failed: Property not found. retry in 643.688368ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:46.961315164Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_33444b97-3e72-47f7-a319-cbd1ca92b6c8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3b0743b7-2816-4de6-a40a-d863e46d23f8\") failed: Property not found. retry in 596.785626ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:46.989455716Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_3b0743b7-2816-4de6-a40a-d863e46d23f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d835fd87-0881-47dd-8e5b-ebb01fd7b0ce\") failed: Property not found. retry in 679.679674ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:47.551124495Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_d835fd87-0881-47dd-8e5b-ebb01fd7b0ce","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3b0743b7-2816-4de6-a40a-d863e46d23f8\") failed: Property not found. retry in 1.076627973s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:47.597443336Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_3b0743b7-2816-4de6-a40a-d863e46d23f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4f6a29ee-ca23-4fa8-be69-497060121028\") failed: Property not found. retry in 675.2868ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:47.608926189Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_4f6a29ee-ca23-4fa8-be69-497060121028","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_33444b97-3e72-47f7-a319-cbd1ca92b6c8\") failed: Property not found. retry in 1.012982768s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:47.616746145Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_33444b97-3e72-47f7-a319-cbd1ca92b6c8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4553d8e1-a7ca-4d20-b1a4-394640a4ad4f\") failed: Property not found. retry in 733.732424ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:47.833167711Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_4553d8e1-a7ca-4d20-b1a4-394640a4ad4f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d835fd87-0881-47dd-8e5b-ebb01fd7b0ce\") failed: Property not found. retry in 834.821527ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:48.242611187Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_d835fd87-0881-47dd-8e5b-ebb01fd7b0ce","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4f6a29ee-ca23-4fa8-be69-497060121028\") failed: Property not found. retry in 1.351120523s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:48.297022029Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_4f6a29ee-ca23-4fa8-be69-497060121028","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4553d8e1-a7ca-4d20-b1a4-394640a4ad4f\") failed: Property not found. retry in 1.236057011s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:48.57829283Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_4553d8e1-a7ca-4d20-b1a4-394640a4ad4f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_33444b97-3e72-47f7-a319-cbd1ca92b6c8\") failed: Property not found. retry in 522.717903ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:48.641234077Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_33444b97-3e72-47f7-a319-cbd1ca92b6c8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3b0743b7-2816-4de6-a40a-d863e46d23f8\") failed: Property not found. retry in 896.52462ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:48.685369386Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_3b0743b7-2816-4de6-a40a-d863e46d23f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d835fd87-0881-47dd-8e5b-ebb01fd7b0ce\") failed: Property not found. retry in 600.910805ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:49.088656864Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_d835fd87-0881-47dd-8e5b-ebb01fd7b0ce","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_33444b97-3e72-47f7-a319-cbd1ca92b6c8\") failed: Property not found. retry in 1.19663353s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:49.174933241Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_33444b97-3e72-47f7-a319-cbd1ca92b6c8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3b0743b7-2816-4de6-a40a-d863e46d23f8\") failed: Property not found. retry in 754.480489ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:49.594672441Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_3b0743b7-2816-4de6-a40a-d863e46d23f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4f6a29ee-ca23-4fa8-be69-497060121028\") failed: Property not found. retry in 778.640707ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:49.662190453Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_4f6a29ee-ca23-4fa8-be69-497060121028","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d835fd87-0881-47dd-8e5b-ebb01fd7b0ce\") failed: Property not found. retry in 1.331698231s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:49.700321215Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_d835fd87-0881-47dd-8e5b-ebb01fd7b0ce","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4553d8e1-a7ca-4d20-b1a4-394640a4ad4f\") failed: Property not found. retry in 658.537264ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:49.826877974Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_4553d8e1-a7ca-4d20-b1a4-394640a4ad4f","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3b0743b7-2816-4de6-a40a-d863e46d23f8\") failed: Property not found. retry in 1.290330039s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:50.362642315Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_3b0743b7-2816-4de6-a40a-d863e46d23f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_33444b97-3e72-47f7-a319-cbd1ca92b6c8\") failed: Property not found. retry in 1.496309738s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:50.383719588Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_33444b97-3e72-47f7-a319-cbd1ca92b6c8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_4f6a29ee-ca23-4fa8-be69-497060121028\") failed: Property not found. retry in 1.438157084s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:50.451193302Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_4f6a29ee-ca23-4fa8-be69-497060121028","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_d835fd87-0881-47dd-8e5b-ebb01fd7b0ce\") failed: Property not found. retry in 1.305752233s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:51.046486496Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_d835fd87-0881-47dd-8e5b-ebb01fd7b0ce","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3b0743b7-2816-4de6-a40a-d863e46d23f8\") failed: Property not found. retry in 680.740592ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:51.666021729Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_3b0743b7-2816-4de6-a40a-d863e46d23f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:51.81514241Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:49883"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:44:51.815324495Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"58b7e820-3a36-40e7-ab67-fbfe0ca61613","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:49883","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":88,"response_code":"200"} {"msg":"wait for service Some(\"oxz_cockroachdb_3b0743b7-2816-4de6-a40a-d863e46d23f8\") failed: Property not found. retry in 889.753104ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:52.358079967Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_3b0743b7-2816-4de6-a40a-d863e46d23f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3b0743b7-2816-4de6-a40a-d863e46d23f8\") failed: Property not found. retry in 1.307726611s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:53.259046554Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_3b0743b7-2816-4de6-a40a-d863e46d23f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3b0743b7-2816-4de6-a40a-d863e46d23f8\") failed: Property not found. retry in 1.07919368s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:54.580276807Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_3b0743b7-2816-4de6-a40a-d863e46d23f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3b0743b7-2816-4de6-a40a-d863e46d23f8\") failed: Property not found. retry in 817.559347ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:55.687403158Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_3b0743b7-2816-4de6-a40a-d863e46d23f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3b0743b7-2816-4de6-a40a-d863e46d23f8\") failed: Property not found. retry in 1.451834132s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:56.517963529Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_3b0743b7-2816-4de6-a40a-d863e46d23f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:56.697280871Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","component":"MetricsManager","sled_id":"8fa4bc78-1026-41f0-b09d-c67224071376","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"488.064022ms"} {"msg":"wait for service Some(\"oxz_cockroachdb_3b0743b7-2816-4de6-a40a-d863e46d23f8\") failed: Property not found. retry in 1.214057976s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:57.984055798Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_3b0743b7-2816-4de6-a40a-d863e46d23f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3b0743b7-2816-4de6-a40a-d863e46d23f8\") failed: Property not found. retry in 798.180637ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:44:59.209685968Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_3b0743b7-2816-4de6-a40a-d863e46d23f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3b0743b7-2816-4de6-a40a-d863e46d23f8\") failed: Property not found. retry in 1.038391511s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:00.020600495Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_3b0743b7-2816-4de6-a40a-d863e46d23f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3b0743b7-2816-4de6-a40a-d863e46d23f8\") failed: Property not found. retry in 625.174019ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:01.072629233Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_3b0743b7-2816-4de6-a40a-d863e46d23f8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:01.720002019Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_4f6a29ee-ca23-4fa8-be69-497060121028"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:01.720076328Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_3b0743b7-2816-4de6-a40a-d863e46d23f8"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:01.720085347Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_d835fd87-0881-47dd-8e5b-ebb01fd7b0ce"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:01.720092875Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_4553d8e1-a7ca-4d20-b1a4-394640a4ad4f"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:01.72010076Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_cockroachdb_33444b97-3e72-47f7-a319-cbd1ca92b6c8"} {"msg":"Writing ledger to /pool/int/6d2aa15e-618e-402e-b368-cf27fd8def68/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:01.720170633Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/75c788e1-45ef-4fcf-958d-db07331bef52/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:01.725787859Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:01.735067683Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"accc0bbb-e5fe-4345-a34b-277637f391e4","remote_addr":"[fd00:1122:3344:101::1]:33707","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":24901126,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:01.767031812Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:63729"} {"msg":"Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::6]:32221","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:01.795913917Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3485"} {"msg":"Formatting CRDB","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:02.554507731Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3503"} {"msg":"Formatting CRDB - Completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:20.833758971Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3526"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:20.833867668Z","hostname":"buskin","pid":1555,"uri":"/cockroachdb","method":"POST","req_id":"5164ed42-f50a-435a-8244-1b5ec385180f","remote_addr":"[fd00:1122:3344:101::1]:63729","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":19015150,"response_code":"204"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:20.838464231Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","component":"MetricsManager","sled_id":"8fa4bc78-1026-41f0-b09d-c67224071376","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"925.847787ms"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3f311cd5-ee38-4707-ab8a-9331491f8e7c, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6389ba5c-9d50-4fe4-b1a2-8be3c6861cb0, kind: Service { id: 3f311cd5-ee38-4707-ab8a-9331491f8e7c }, name: Name(\"ntp-3f311cd5-ee38-4707-ab8a-9331491f8e7c\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 235])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4f6a29ee-ca23-4fa8-be69-497060121028, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } } } }, OmicronZoneConfig { id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } } } }, OmicronZoneConfig { id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External } } } }, OmicronZoneConfig { id: b04db43b-dc24-43ce-9d97-32193f88e046, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f04fae05-ec27-43ac-ad4d-ab63c091bf1b, kind: Service { id: b04db43b-dc24-43ce-9d97-32193f88e046 }, name: Name(\"external-dns-b04db43b-dc24-43ce-9d97-32193f88e046\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 154])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 2c3f8453-233f-428f-93cc-0bfefb84929d, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: a6846ef5-8686-4c7e-ae28-d178a4029a85, kind: Service { id: 2c3f8453-233f-428f-93cc-0bfefb84929d }, name: Name(\"external-dns-2c3f8453-233f-428f-93cc-0bfefb84929d\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 164, 190])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: e4efbc31-e513-4290-82f5-c51d8afea0c7, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 735e2401-2ba5-4079-b4a0-7f0184716575, kind: Service { id: e4efbc31-e513-4290-82f5-c51d8afea0c7 }, name: Name(\"nexus-e4efbc31-e513-4290-82f5-c51d8afea0c7\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 154])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 85472347-dfd9-425e-bd98-9b81ebdf3b2b, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: f33554f0-0c2d-4b66-98d0-73365d475c4b, kind: Service { id: 85472347-dfd9-425e-bd98-9b81ebdf3b2b }, name: Name(\"nexus-85472347-dfd9-425e-bd98-9b81ebdf3b2b\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 244])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 221c5e7c-0832-4890-8b0a-f9ba08725876, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 9e8859b7-3d6e-4066-b58c-24fc535c1122, kind: Service { id: 221c5e7c-0832-4890-8b0a-f9ba08725876 }, name: Name(\"nexus-221c5e7c-0832-4890-8b0a-f9ba08725876\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 155])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4b4998f4-10de-49f2-9326-b482137c5fd3, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: a635cc62-7516-4fd9-b490-62d6de386095, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: ac9fa851-e7e1-4295-ac79-d6076526f465, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 1cb96976-2047-44b8-a79c-9fcd74d17bfa, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: a0b9fba0-c330-402f-9bc0-a7510786c98a, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } } } }, OmicronZoneConfig { id: d94d9602-d610-44ce-b9fc-06ded41a1480, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } } } }, OmicronZoneConfig { id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External } } } }, OmicronZoneConfig { id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External } } } }, OmicronZoneConfig { id: dfb1f626-0f55-4385-87de-de59052d98d2, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External } } } }, OmicronZoneConfig { id: adff3617-5320-4e42-8b10-1430a1eb0afc, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:20.868635823Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:434"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:20.868812067Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:47495"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:20.86918296Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:20.869188474Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:20.890465484Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:20.890587263Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:20.912035907Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:20.912086177Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4f6a29ee-ca23-4fa8-be69-497060121028, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:20.932912179Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:20.93295867Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4f6a29ee-ca23-4fa8-be69-497060121028, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:20.953173163Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:20.953219185Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:20.972383776Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:20.972427614Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:20.992852927Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:20.992892369Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:21.012930854Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:21.012983336Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b04db43b-dc24-43ce-9d97-32193f88e046, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:21.03529003Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:21.035332772Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: b04db43b-dc24-43ce-9d97-32193f88e046, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2c3f8453-233f-428f-93cc-0bfefb84929d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:21.155952085Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:21.156005242Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2c3f8453-233f-428f-93cc-0bfefb84929d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:21.257389674Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:21.257463259Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a0b9fba0-c330-402f-9bc0-a7510786c98a, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:21.380492229Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:21.380542217Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: a0b9fba0-c330-402f-9bc0-a7510786c98a, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:21.48571674Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:21.485767779Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d94d9602-d610-44ce-b9fc-06ded41a1480, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:21.600254619Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:21.600308967Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d94d9602-d610-44ce-b9fc-06ded41a1480, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:21.717410256Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:21.717464557Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:21.870249843Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:21.870314617Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:21.931303054Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:37914"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:21.931472413Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"97226c11-239c-4ed3-8ee5-62de403a8c5b","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:37914","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":59,"response_code":"200"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, dataset_name: DatasetName { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:22.013291956Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:22.013347607Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, dataset_name: DatasetName { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dfb1f626-0f55-4385-87de-de59052d98d2, dataset_name: DatasetName { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:22.134607903Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:22.134653392Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: dfb1f626-0f55-4385-87de-de59052d98d2, dataset_name: DatasetName { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: adff3617-5320-4e42-8b10-1430a1eb0afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:22.257463908Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:22.25751549Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: adff3617-5320-4e42-8b10-1430a1eb0afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, dataset_name: DatasetName { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:22.354829594Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:22.354875729Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, dataset_name: DatasetName { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:22.451973205Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/6d2aa15e-618e-402e-b368-cf27fd8def68/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:22.452030384Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/75c788e1-45ef-4fcf-958d-db07331bef52/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:22.452317856Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:22.452496107Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:22.552556742Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:22.552597936Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:22.552605343Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:22.552610387Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:22.552615421Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:22.552620494Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:22.552626175Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:22.552641175Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:22.552646397Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:22.552652088Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:22.552656615Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:22.552661416Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:22.552667051Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:22.552672966Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:22.552677607Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:22.552682276Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:22.552687798Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:22.552693527Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:22.552697342Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:22.552783332Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"691a1dfe-1f82-42d7-8d71-6e986e50417f","remote_addr":"[fd00:1122:3344:101::1]:47495","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2\", TimeNotSynchronized), (\"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9\", TimeNotSynchronized), (\"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480\", TimeNotSynchronized), (\"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046\", TimeNotSynchronized), (\"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e\", TimeNotSynchronized), (\"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b\", TimeNotSynchronized), (\"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa\", TimeNotSynchronized), (\"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc\", TimeNotSynchronized), (\"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7\", TimeNotSynchronized), (\"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c\", TimeNotSynchronized), (\"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465\", TimeNotSynchronized), (\"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79\", TimeNotSynchronized), (\"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d\", TimeNotSynchronized), (\"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876\", TimeNotSynchronized), (\"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862\", TimeNotSynchronized), (\"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3\", TimeNotSynchronized), (\"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095\", TimeNotSynchronized), (\"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a\", TimeNotSynchronized), (\"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c\", TimeNotSynchronized)]","latency_us":1679288,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:22.553114831Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:472","retry_after":"107.192945ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"691a1dfe-1f82-42d7-8d71-6e986e50417f\", \"content-length\": \"124\", \"date\": \"Thu, 25 Jul 2024 05:45:21 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"691a1dfe-1f82-42d7-8d71-6e986e50417f\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3f311cd5-ee38-4707-ab8a-9331491f8e7c, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6389ba5c-9d50-4fe4-b1a2-8be3c6861cb0, kind: Service { id: 3f311cd5-ee38-4707-ab8a-9331491f8e7c }, name: Name(\"ntp-3f311cd5-ee38-4707-ab8a-9331491f8e7c\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 235])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4f6a29ee-ca23-4fa8-be69-497060121028, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } } } }, OmicronZoneConfig { id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } } } }, OmicronZoneConfig { id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External } } } }, OmicronZoneConfig { id: b04db43b-dc24-43ce-9d97-32193f88e046, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f04fae05-ec27-43ac-ad4d-ab63c091bf1b, kind: Service { id: b04db43b-dc24-43ce-9d97-32193f88e046 }, name: Name(\"external-dns-b04db43b-dc24-43ce-9d97-32193f88e046\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 154])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 2c3f8453-233f-428f-93cc-0bfefb84929d, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: a6846ef5-8686-4c7e-ae28-d178a4029a85, kind: Service { id: 2c3f8453-233f-428f-93cc-0bfefb84929d }, name: Name(\"external-dns-2c3f8453-233f-428f-93cc-0bfefb84929d\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 164, 190])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: e4efbc31-e513-4290-82f5-c51d8afea0c7, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 735e2401-2ba5-4079-b4a0-7f0184716575, kind: Service { id: e4efbc31-e513-4290-82f5-c51d8afea0c7 }, name: Name(\"nexus-e4efbc31-e513-4290-82f5-c51d8afea0c7\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 154])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 85472347-dfd9-425e-bd98-9b81ebdf3b2b, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: f33554f0-0c2d-4b66-98d0-73365d475c4b, kind: Service { id: 85472347-dfd9-425e-bd98-9b81ebdf3b2b }, name: Name(\"nexus-85472347-dfd9-425e-bd98-9b81ebdf3b2b\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 244])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 221c5e7c-0832-4890-8b0a-f9ba08725876, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 9e8859b7-3d6e-4066-b58c-24fc535c1122, kind: Service { id: 221c5e7c-0832-4890-8b0a-f9ba08725876 }, name: Name(\"nexus-221c5e7c-0832-4890-8b0a-f9ba08725876\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 155])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4b4998f4-10de-49f2-9326-b482137c5fd3, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: a635cc62-7516-4fd9-b490-62d6de386095, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: ac9fa851-e7e1-4295-ac79-d6076526f465, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 1cb96976-2047-44b8-a79c-9fcd74d17bfa, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: a0b9fba0-c330-402f-9bc0-a7510786c98a, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } } } }, OmicronZoneConfig { id: d94d9602-d610-44ce-b9fc-06ded41a1480, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } } } }, OmicronZoneConfig { id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External } } } }, OmicronZoneConfig { id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External } } } }, OmicronZoneConfig { id: dfb1f626-0f55-4385-87de-de59052d98d2, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External } } } }, OmicronZoneConfig { id: adff3617-5320-4e42-8b10-1430a1eb0afc, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:22.661821625Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:434"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:22.662274936Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:22.662281986Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:22.684527972Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:22.684582104Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:22.704190327Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:22.704230163Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4f6a29ee-ca23-4fa8-be69-497060121028, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:22.724542196Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:22.724582772Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4f6a29ee-ca23-4fa8-be69-497060121028, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:22.742886856Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:22.742930057Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:22.760799498Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:22.76083938Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:22.780981293Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:22.781031114Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:22.801514889Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:22.801561765Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b04db43b-dc24-43ce-9d97-32193f88e046, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:22.821322712Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:22.821366616Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: b04db43b-dc24-43ce-9d97-32193f88e046, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2c3f8453-233f-428f-93cc-0bfefb84929d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:22.840719717Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:22.840765037Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2c3f8453-233f-428f-93cc-0bfefb84929d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:22.860047792Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:22.860099159Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a0b9fba0-c330-402f-9bc0-a7510786c98a, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:22.878542705Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:22.878585822Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: a0b9fba0-c330-402f-9bc0-a7510786c98a, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:22.900822335Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:22.90086728Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d94d9602-d610-44ce-b9fc-06ded41a1480, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:22.92261557Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:22.92265654Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d94d9602-d610-44ce-b9fc-06ded41a1480, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:22.942581389Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:22.94262444Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:22.963303372Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:22.963354466Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, dataset_name: DatasetName { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:22.982471602Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:22.982513715Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, dataset_name: DatasetName { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dfb1f626-0f55-4385-87de-de59052d98d2, dataset_name: DatasetName { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:23.002276746Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:23.002322178Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: dfb1f626-0f55-4385-87de-de59052d98d2, dataset_name: DatasetName { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: adff3617-5320-4e42-8b10-1430a1eb0afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:23.02271523Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:23.022758928Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: adff3617-5320-4e42-8b10-1430a1eb0afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, dataset_name: DatasetName { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:23.042385751Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:23.04243019Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, dataset_name: DatasetName { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:23.061150468Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/6d2aa15e-618e-402e-b368-cf27fd8def68/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:23.061208172Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/75c788e1-45ef-4fcf-958d-db07331bef52/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:23.061507442Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:23.061635321Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:23.156588501Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:23.156706624Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:23.156712727Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:23.15671688Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:23.156721183Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:23.156725225Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:23.156729883Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:23.156732987Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:23.156738678Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:23.156742981Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:23.156746112Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:23.156759275Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:23.156763372Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:23.156768069Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:23.156771154Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:23.156775823Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:23.156778973Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:23.156783548Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:23.156787344Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:23.156893041Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"c25a7f8f-6a51-48a0-a5a1-63a7a430a1cc","remote_addr":"[fd00:1122:3344:101::1]:47495","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79\", TimeNotSynchronized), (\"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a\", TimeNotSynchronized), (\"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa\", TimeNotSynchronized), (\"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc\", TimeNotSynchronized), (\"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e\", TimeNotSynchronized), (\"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c\", TimeNotSynchronized), (\"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862\", TimeNotSynchronized), (\"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7\", TimeNotSynchronized), (\"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9\", TimeNotSynchronized), (\"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465\", TimeNotSynchronized), (\"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876\", TimeNotSynchronized), (\"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3\", TimeNotSynchronized), (\"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c\", TimeNotSynchronized), (\"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046\", TimeNotSynchronized), (\"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d\", TimeNotSynchronized), (\"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b\", TimeNotSynchronized), (\"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095\", TimeNotSynchronized), (\"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480\", TimeNotSynchronized), (\"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2\", TimeNotSynchronized)]","latency_us":493429,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:23.157065676Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:472","retry_after":"76.267732ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c25a7f8f-6a51-48a0-a5a1-63a7a430a1cc\", \"content-length\": \"124\", \"date\": \"Thu, 25 Jul 2024 05:45:23 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"c25a7f8f-6a51-48a0-a5a1-63a7a430a1cc\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3f311cd5-ee38-4707-ab8a-9331491f8e7c, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6389ba5c-9d50-4fe4-b1a2-8be3c6861cb0, kind: Service { id: 3f311cd5-ee38-4707-ab8a-9331491f8e7c }, name: Name(\"ntp-3f311cd5-ee38-4707-ab8a-9331491f8e7c\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 235])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4f6a29ee-ca23-4fa8-be69-497060121028, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } } } }, OmicronZoneConfig { id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } } } }, OmicronZoneConfig { id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External } } } }, OmicronZoneConfig { id: b04db43b-dc24-43ce-9d97-32193f88e046, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f04fae05-ec27-43ac-ad4d-ab63c091bf1b, kind: Service { id: b04db43b-dc24-43ce-9d97-32193f88e046 }, name: Name(\"external-dns-b04db43b-dc24-43ce-9d97-32193f88e046\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 154])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 2c3f8453-233f-428f-93cc-0bfefb84929d, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: a6846ef5-8686-4c7e-ae28-d178a4029a85, kind: Service { id: 2c3f8453-233f-428f-93cc-0bfefb84929d }, name: Name(\"external-dns-2c3f8453-233f-428f-93cc-0bfefb84929d\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 164, 190])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: e4efbc31-e513-4290-82f5-c51d8afea0c7, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 735e2401-2ba5-4079-b4a0-7f0184716575, kind: Service { id: e4efbc31-e513-4290-82f5-c51d8afea0c7 }, name: Name(\"nexus-e4efbc31-e513-4290-82f5-c51d8afea0c7\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 154])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 85472347-dfd9-425e-bd98-9b81ebdf3b2b, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: f33554f0-0c2d-4b66-98d0-73365d475c4b, kind: Service { id: 85472347-dfd9-425e-bd98-9b81ebdf3b2b }, name: Name(\"nexus-85472347-dfd9-425e-bd98-9b81ebdf3b2b\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 244])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 221c5e7c-0832-4890-8b0a-f9ba08725876, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 9e8859b7-3d6e-4066-b58c-24fc535c1122, kind: Service { id: 221c5e7c-0832-4890-8b0a-f9ba08725876 }, name: Name(\"nexus-221c5e7c-0832-4890-8b0a-f9ba08725876\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 155])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4b4998f4-10de-49f2-9326-b482137c5fd3, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: a635cc62-7516-4fd9-b490-62d6de386095, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: ac9fa851-e7e1-4295-ac79-d6076526f465, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 1cb96976-2047-44b8-a79c-9fcd74d17bfa, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: a0b9fba0-c330-402f-9bc0-a7510786c98a, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } } } }, OmicronZoneConfig { id: d94d9602-d610-44ce-b9fc-06ded41a1480, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } } } }, OmicronZoneConfig { id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External } } } }, OmicronZoneConfig { id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External } } } }, OmicronZoneConfig { id: dfb1f626-0f55-4385-87de-de59052d98d2, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External } } } }, OmicronZoneConfig { id: adff3617-5320-4e42-8b10-1430a1eb0afc, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:23.23546358Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:434"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:23.235772429Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:23.235783023Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:23.258543063Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:23.258582092Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:23.279222071Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:23.279265487Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4f6a29ee-ca23-4fa8-be69-497060121028, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:23.298591349Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:23.298621894Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4f6a29ee-ca23-4fa8-be69-497060121028, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:23.319278617Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:23.319311289Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:23.33845953Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:23.338488846Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:23.356937394Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:23.356968979Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:23.375005768Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:23.375042322Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b04db43b-dc24-43ce-9d97-32193f88e046, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:23.394035494Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:23.394071551Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: b04db43b-dc24-43ce-9d97-32193f88e046, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2c3f8453-233f-428f-93cc-0bfefb84929d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:23.413314748Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:23.413416505Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2c3f8453-233f-428f-93cc-0bfefb84929d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:23.432243703Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:23.432275776Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a0b9fba0-c330-402f-9bc0-a7510786c98a, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:23.451191515Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:23.451227956Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: a0b9fba0-c330-402f-9bc0-a7510786c98a, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:23.470223662Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:23.470250897Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d94d9602-d610-44ce-b9fc-06ded41a1480, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:23.492104367Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:23.49213853Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d94d9602-d610-44ce-b9fc-06ded41a1480, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:23.515052161Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:23.515087814Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:23.536317314Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:23.536350024Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, dataset_name: DatasetName { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:23.556694461Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:23.556797805Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, dataset_name: DatasetName { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dfb1f626-0f55-4385-87de-de59052d98d2, dataset_name: DatasetName { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:23.577131231Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:23.577160857Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: dfb1f626-0f55-4385-87de-de59052d98d2, dataset_name: DatasetName { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: adff3617-5320-4e42-8b10-1430a1eb0afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:23.597313488Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:23.59734841Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: adff3617-5320-4e42-8b10-1430a1eb0afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, dataset_name: DatasetName { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:23.617353233Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:23.617380628Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, dataset_name: DatasetName { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:23.637185502Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/6d2aa15e-618e-402e-b368-cf27fd8def68/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:23.637226763Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/75c788e1-45ef-4fcf-958d-db07331bef52/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:23.637396989Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:23.637495897Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:23.733067949Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:23.733122692Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:23.733130042Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:23.733136108Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:23.733140467Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:23.733144686Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:23.733151998Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:23.733157746Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:23.733165293Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:23.733171387Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:23.733177471Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:23.733181849Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:23.733253667Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:23.733260023Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:23.733267393Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:23.733281324Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:23.733288777Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:23.733294252Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:23.733299999Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:23.733398074Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"dea6787a-2fba-4c76-ac82-dc1c0043f9ec","remote_addr":"[fd00:1122:3344:101::1]:47495","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046\", TimeNotSynchronized), (\"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2\", TimeNotSynchronized), (\"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862\", TimeNotSynchronized), (\"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc\", TimeNotSynchronized), (\"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d\", TimeNotSynchronized), (\"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3\", TimeNotSynchronized), (\"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a\", TimeNotSynchronized), (\"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c\", TimeNotSynchronized), (\"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b\", TimeNotSynchronized), (\"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa\", TimeNotSynchronized), (\"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480\", TimeNotSynchronized), (\"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095\", TimeNotSynchronized), (\"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79\", TimeNotSynchronized), (\"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7\", TimeNotSynchronized), (\"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465\", TimeNotSynchronized), (\"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c\", TimeNotSynchronized), (\"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876\", TimeNotSynchronized), (\"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e\", TimeNotSynchronized), (\"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9\", TimeNotSynchronized)]","latency_us":496421,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:23.733601722Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:472","retry_after":"115.119741ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"dea6787a-2fba-4c76-ac82-dc1c0043f9ec\", \"content-length\": \"124\", \"date\": \"Thu, 25 Jul 2024 05:45:23 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"dea6787a-2fba-4c76-ac82-dc1c0043f9ec\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3f311cd5-ee38-4707-ab8a-9331491f8e7c, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6389ba5c-9d50-4fe4-b1a2-8be3c6861cb0, kind: Service { id: 3f311cd5-ee38-4707-ab8a-9331491f8e7c }, name: Name(\"ntp-3f311cd5-ee38-4707-ab8a-9331491f8e7c\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 235])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4f6a29ee-ca23-4fa8-be69-497060121028, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } } } }, OmicronZoneConfig { id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } } } }, OmicronZoneConfig { id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External } } } }, OmicronZoneConfig { id: b04db43b-dc24-43ce-9d97-32193f88e046, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f04fae05-ec27-43ac-ad4d-ab63c091bf1b, kind: Service { id: b04db43b-dc24-43ce-9d97-32193f88e046 }, name: Name(\"external-dns-b04db43b-dc24-43ce-9d97-32193f88e046\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 154])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 2c3f8453-233f-428f-93cc-0bfefb84929d, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: a6846ef5-8686-4c7e-ae28-d178a4029a85, kind: Service { id: 2c3f8453-233f-428f-93cc-0bfefb84929d }, name: Name(\"external-dns-2c3f8453-233f-428f-93cc-0bfefb84929d\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 164, 190])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: e4efbc31-e513-4290-82f5-c51d8afea0c7, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 735e2401-2ba5-4079-b4a0-7f0184716575, kind: Service { id: e4efbc31-e513-4290-82f5-c51d8afea0c7 }, name: Name(\"nexus-e4efbc31-e513-4290-82f5-c51d8afea0c7\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 154])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 85472347-dfd9-425e-bd98-9b81ebdf3b2b, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: f33554f0-0c2d-4b66-98d0-73365d475c4b, kind: Service { id: 85472347-dfd9-425e-bd98-9b81ebdf3b2b }, name: Name(\"nexus-85472347-dfd9-425e-bd98-9b81ebdf3b2b\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 244])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 221c5e7c-0832-4890-8b0a-f9ba08725876, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 9e8859b7-3d6e-4066-b58c-24fc535c1122, kind: Service { id: 221c5e7c-0832-4890-8b0a-f9ba08725876 }, name: Name(\"nexus-221c5e7c-0832-4890-8b0a-f9ba08725876\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 155])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4b4998f4-10de-49f2-9326-b482137c5fd3, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: a635cc62-7516-4fd9-b490-62d6de386095, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: ac9fa851-e7e1-4295-ac79-d6076526f465, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 1cb96976-2047-44b8-a79c-9fcd74d17bfa, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: a0b9fba0-c330-402f-9bc0-a7510786c98a, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } } } }, OmicronZoneConfig { id: d94d9602-d610-44ce-b9fc-06ded41a1480, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } } } }, OmicronZoneConfig { id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External } } } }, OmicronZoneConfig { id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External } } } }, OmicronZoneConfig { id: dfb1f626-0f55-4385-87de-de59052d98d2, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External } } } }, OmicronZoneConfig { id: adff3617-5320-4e42-8b10-1430a1eb0afc, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:23.849320069Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:434"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:23.84967246Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:23.849682379Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:23.872658073Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:23.87269758Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:23.894350231Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:23.894390882Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4f6a29ee-ca23-4fa8-be69-497060121028, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:23.916880304Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:23.916931269Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4f6a29ee-ca23-4fa8-be69-497060121028, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:23.938187026Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:23.938223965Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:23.957991211Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:23.958026716Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:23.976642833Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:23.976681402Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:23.995879432Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:23.995982444Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b04db43b-dc24-43ce-9d97-32193f88e046, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:24.015338501Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:24.015384552Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: b04db43b-dc24-43ce-9d97-32193f88e046, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2c3f8453-233f-428f-93cc-0bfefb84929d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:24.033427446Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:24.033461759Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2c3f8453-233f-428f-93cc-0bfefb84929d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:24.05368602Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:24.053721909Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a0b9fba0-c330-402f-9bc0-a7510786c98a, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:24.07350494Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:24.073540669Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: a0b9fba0-c330-402f-9bc0-a7510786c98a, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:24.092915918Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:24.092955069Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d94d9602-d610-44ce-b9fc-06ded41a1480, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:24.111486348Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:24.111518805Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d94d9602-d610-44ce-b9fc-06ded41a1480, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:24.131992477Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:24.132030136Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:24.150475427Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:24.150509769Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, dataset_name: DatasetName { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:24.168348273Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:24.168446063Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, dataset_name: DatasetName { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dfb1f626-0f55-4385-87de-de59052d98d2, dataset_name: DatasetName { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:24.186192648Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:24.186243256Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: dfb1f626-0f55-4385-87de-de59052d98d2, dataset_name: DatasetName { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: adff3617-5320-4e42-8b10-1430a1eb0afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:24.2047078Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:24.204754301Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: adff3617-5320-4e42-8b10-1430a1eb0afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, dataset_name: DatasetName { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:24.223576731Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:24.223615254Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, dataset_name: DatasetName { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:24.241840833Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/6d2aa15e-618e-402e-b368-cf27fd8def68/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:24.241882759Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/75c788e1-45ef-4fcf-958d-db07331bef52/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:24.242098079Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:24.242202669Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:24.34030836Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:24.340351261Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:24.34035728Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:24.340361414Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:24.340380043Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:24.340384215Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:24.340387243Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:24.340390224Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:24.340393178Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:24.340396787Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:24.340399703Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:24.340403368Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:24.340407174Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:24.340410128Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:24.340412969Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:24.340415847Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:24.340419457Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:24.340423403Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:24.340426394Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:24.340460923Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"4fe74a12-ba05-4cd8-9d0b-9875e61ad626","remote_addr":"[fd00:1122:3344:101::1]:47495","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480\", TimeNotSynchronized), (\"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7\", TimeNotSynchronized), (\"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095\", TimeNotSynchronized), (\"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3\", TimeNotSynchronized), (\"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d\", TimeNotSynchronized), (\"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876\", TimeNotSynchronized), (\"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465\", TimeNotSynchronized), (\"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a\", TimeNotSynchronized), (\"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c\", TimeNotSynchronized), (\"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046\", TimeNotSynchronized), (\"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e\", TimeNotSynchronized), (\"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c\", TimeNotSynchronized), (\"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc\", TimeNotSynchronized), (\"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9\", TimeNotSynchronized), (\"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862\", TimeNotSynchronized), (\"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa\", TimeNotSynchronized), (\"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79\", TimeNotSynchronized), (\"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2\", TimeNotSynchronized), (\"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b\", TimeNotSynchronized)]","latency_us":489672,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:24.340627071Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:472","retry_after":"153.645353ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"4fe74a12-ba05-4cd8-9d0b-9875e61ad626\", \"content-length\": \"124\", \"date\": \"Thu, 25 Jul 2024 05:45:24 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"4fe74a12-ba05-4cd8-9d0b-9875e61ad626\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3f311cd5-ee38-4707-ab8a-9331491f8e7c, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6389ba5c-9d50-4fe4-b1a2-8be3c6861cb0, kind: Service { id: 3f311cd5-ee38-4707-ab8a-9331491f8e7c }, name: Name(\"ntp-3f311cd5-ee38-4707-ab8a-9331491f8e7c\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 235])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4f6a29ee-ca23-4fa8-be69-497060121028, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } } } }, OmicronZoneConfig { id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } } } }, OmicronZoneConfig { id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External } } } }, OmicronZoneConfig { id: b04db43b-dc24-43ce-9d97-32193f88e046, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f04fae05-ec27-43ac-ad4d-ab63c091bf1b, kind: Service { id: b04db43b-dc24-43ce-9d97-32193f88e046 }, name: Name(\"external-dns-b04db43b-dc24-43ce-9d97-32193f88e046\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 154])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 2c3f8453-233f-428f-93cc-0bfefb84929d, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: a6846ef5-8686-4c7e-ae28-d178a4029a85, kind: Service { id: 2c3f8453-233f-428f-93cc-0bfefb84929d }, name: Name(\"external-dns-2c3f8453-233f-428f-93cc-0bfefb84929d\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 164, 190])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: e4efbc31-e513-4290-82f5-c51d8afea0c7, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 735e2401-2ba5-4079-b4a0-7f0184716575, kind: Service { id: e4efbc31-e513-4290-82f5-c51d8afea0c7 }, name: Name(\"nexus-e4efbc31-e513-4290-82f5-c51d8afea0c7\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 154])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 85472347-dfd9-425e-bd98-9b81ebdf3b2b, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: f33554f0-0c2d-4b66-98d0-73365d475c4b, kind: Service { id: 85472347-dfd9-425e-bd98-9b81ebdf3b2b }, name: Name(\"nexus-85472347-dfd9-425e-bd98-9b81ebdf3b2b\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 244])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 221c5e7c-0832-4890-8b0a-f9ba08725876, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 9e8859b7-3d6e-4066-b58c-24fc535c1122, kind: Service { id: 221c5e7c-0832-4890-8b0a-f9ba08725876 }, name: Name(\"nexus-221c5e7c-0832-4890-8b0a-f9ba08725876\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 155])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4b4998f4-10de-49f2-9326-b482137c5fd3, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: a635cc62-7516-4fd9-b490-62d6de386095, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: ac9fa851-e7e1-4295-ac79-d6076526f465, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 1cb96976-2047-44b8-a79c-9fcd74d17bfa, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: a0b9fba0-c330-402f-9bc0-a7510786c98a, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } } } }, OmicronZoneConfig { id: d94d9602-d610-44ce-b9fc-06ded41a1480, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } } } }, OmicronZoneConfig { id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External } } } }, OmicronZoneConfig { id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External } } } }, OmicronZoneConfig { id: dfb1f626-0f55-4385-87de-de59052d98d2, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External } } } }, OmicronZoneConfig { id: adff3617-5320-4e42-8b10-1430a1eb0afc, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:24.495721266Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:434"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:24.496027488Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:24.496034229Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:24.519175538Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:24.519199885Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:24.539888139Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:24.539913021Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4f6a29ee-ca23-4fa8-be69-497060121028, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:24.558234872Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:24.558260017Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4f6a29ee-ca23-4fa8-be69-497060121028, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:24.577358899Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:24.577388065Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:24.596208654Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:24.596234848Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:24.616431375Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:24.616458929Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:24.636770729Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:24.636799287Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b04db43b-dc24-43ce-9d97-32193f88e046, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:24.652860808Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:24.652890321Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: b04db43b-dc24-43ce-9d97-32193f88e046, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2c3f8453-233f-428f-93cc-0bfefb84929d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:24.669865037Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:24.669897644Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2c3f8453-233f-428f-93cc-0bfefb84929d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:24.688962297Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:24.688983982Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a0b9fba0-c330-402f-9bc0-a7510786c98a, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:24.708091628Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:24.708112338Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: a0b9fba0-c330-402f-9bc0-a7510786c98a, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:24.726624602Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:24.726651603Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d94d9602-d610-44ce-b9fc-06ded41a1480, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:24.746592941Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:24.74661472Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d94d9602-d610-44ce-b9fc-06ded41a1480, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:24.766712797Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:24.766734687Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:24.786368356Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:24.786392703Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, dataset_name: DatasetName { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:24.805532974Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:24.805556056Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, dataset_name: DatasetName { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dfb1f626-0f55-4385-87de-de59052d98d2, dataset_name: DatasetName { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:24.824540995Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:24.824566815Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: dfb1f626-0f55-4385-87de-de59052d98d2, dataset_name: DatasetName { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: adff3617-5320-4e42-8b10-1430a1eb0afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:24.845260503Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:24.845290644Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: adff3617-5320-4e42-8b10-1430a1eb0afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, dataset_name: DatasetName { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:24.863887516Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:24.863910916Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, dataset_name: DatasetName { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:24.882684361Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/6d2aa15e-618e-402e-b368-cf27fd8def68/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:24.882716958Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/75c788e1-45ef-4fcf-958d-db07331bef52/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:24.882899185Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:24.882986955Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:24.975787653Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:24.97584682Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:24.975853786Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:24.975859401Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:24.975864586Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:24.975871186Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:24.975876746Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:24.975899724Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:24.975905546Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:24.97591088Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:24.975917753Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:24.975923303Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:24.97592845Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:24.975933803Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:24.975940459Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:24.975944341Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:24.975949684Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:24.975954467Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:24.975962567Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:24.976093651Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"dafd868d-d39f-4a0e-88ad-63b01a7b10dc","remote_addr":"[fd00:1122:3344:101::1]:47495","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e\", TimeNotSynchronized), (\"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2\", TimeNotSynchronized), (\"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876\", TimeNotSynchronized), (\"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa\", TimeNotSynchronized), (\"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095\", TimeNotSynchronized), (\"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c\", TimeNotSynchronized), (\"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc\", TimeNotSynchronized), (\"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9\", TimeNotSynchronized), (\"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480\", TimeNotSynchronized), (\"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3\", TimeNotSynchronized), (\"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465\", TimeNotSynchronized), (\"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c\", TimeNotSynchronized), (\"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7\", TimeNotSynchronized), (\"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d\", TimeNotSynchronized), (\"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b\", TimeNotSynchronized), (\"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862\", TimeNotSynchronized), (\"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a\", TimeNotSynchronized), (\"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046\", TimeNotSynchronized), (\"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79\", TimeNotSynchronized)]","latency_us":478944,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:24.97628228Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:472","retry_after":"111.028637ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"dafd868d-d39f-4a0e-88ad-63b01a7b10dc\", \"content-length\": \"124\", \"date\": \"Thu, 25 Jul 2024 05:45:24 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"dafd868d-d39f-4a0e-88ad-63b01a7b10dc\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3f311cd5-ee38-4707-ab8a-9331491f8e7c, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6389ba5c-9d50-4fe4-b1a2-8be3c6861cb0, kind: Service { id: 3f311cd5-ee38-4707-ab8a-9331491f8e7c }, name: Name(\"ntp-3f311cd5-ee38-4707-ab8a-9331491f8e7c\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 235])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4f6a29ee-ca23-4fa8-be69-497060121028, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } } } }, OmicronZoneConfig { id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } } } }, OmicronZoneConfig { id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External } } } }, OmicronZoneConfig { id: b04db43b-dc24-43ce-9d97-32193f88e046, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f04fae05-ec27-43ac-ad4d-ab63c091bf1b, kind: Service { id: b04db43b-dc24-43ce-9d97-32193f88e046 }, name: Name(\"external-dns-b04db43b-dc24-43ce-9d97-32193f88e046\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 154])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 2c3f8453-233f-428f-93cc-0bfefb84929d, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: a6846ef5-8686-4c7e-ae28-d178a4029a85, kind: Service { id: 2c3f8453-233f-428f-93cc-0bfefb84929d }, name: Name(\"external-dns-2c3f8453-233f-428f-93cc-0bfefb84929d\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 164, 190])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: e4efbc31-e513-4290-82f5-c51d8afea0c7, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 735e2401-2ba5-4079-b4a0-7f0184716575, kind: Service { id: e4efbc31-e513-4290-82f5-c51d8afea0c7 }, name: Name(\"nexus-e4efbc31-e513-4290-82f5-c51d8afea0c7\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 154])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 85472347-dfd9-425e-bd98-9b81ebdf3b2b, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: f33554f0-0c2d-4b66-98d0-73365d475c4b, kind: Service { id: 85472347-dfd9-425e-bd98-9b81ebdf3b2b }, name: Name(\"nexus-85472347-dfd9-425e-bd98-9b81ebdf3b2b\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 244])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 221c5e7c-0832-4890-8b0a-f9ba08725876, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 9e8859b7-3d6e-4066-b58c-24fc535c1122, kind: Service { id: 221c5e7c-0832-4890-8b0a-f9ba08725876 }, name: Name(\"nexus-221c5e7c-0832-4890-8b0a-f9ba08725876\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 155])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4b4998f4-10de-49f2-9326-b482137c5fd3, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: a635cc62-7516-4fd9-b490-62d6de386095, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: ac9fa851-e7e1-4295-ac79-d6076526f465, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 1cb96976-2047-44b8-a79c-9fcd74d17bfa, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: a0b9fba0-c330-402f-9bc0-a7510786c98a, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } } } }, OmicronZoneConfig { id: d94d9602-d610-44ce-b9fc-06ded41a1480, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } } } }, OmicronZoneConfig { id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External } } } }, OmicronZoneConfig { id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External } } } }, OmicronZoneConfig { id: dfb1f626-0f55-4385-87de-de59052d98d2, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External } } } }, OmicronZoneConfig { id: adff3617-5320-4e42-8b10-1430a1eb0afc, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:25.088777769Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:434"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:25.089153532Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:25.089160638Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:25.112893702Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:25.113002257Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:25.134773188Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:25.134808739Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4f6a29ee-ca23-4fa8-be69-497060121028, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:25.154358411Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:25.154400937Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4f6a29ee-ca23-4fa8-be69-497060121028, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:25.175602587Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:25.175641532Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:25.193876133Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:25.193915462Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:25.211828537Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:25.2118732Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:25.229844451Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:25.229887277Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b04db43b-dc24-43ce-9d97-32193f88e046, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:25.252206437Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:25.252255872Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: b04db43b-dc24-43ce-9d97-32193f88e046, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2c3f8453-233f-428f-93cc-0bfefb84929d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:25.27276606Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:25.272807161Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2c3f8453-233f-428f-93cc-0bfefb84929d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:25.293144663Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:25.293192486Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a0b9fba0-c330-402f-9bc0-a7510786c98a, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:25.312534651Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:25.312585773Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: a0b9fba0-c330-402f-9bc0-a7510786c98a, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:25.334268875Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:25.334314082Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d94d9602-d610-44ce-b9fc-06ded41a1480, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:25.35430646Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:25.354349089Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d94d9602-d610-44ce-b9fc-06ded41a1480, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:25.37599102Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:25.376034296Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:25.396649241Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:25.396782725Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, dataset_name: DatasetName { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:25.416594572Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:25.416640454Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, dataset_name: DatasetName { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dfb1f626-0f55-4385-87de-de59052d98d2, dataset_name: DatasetName { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:25.436170908Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:25.436275085Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: dfb1f626-0f55-4385-87de-de59052d98d2, dataset_name: DatasetName { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: adff3617-5320-4e42-8b10-1430a1eb0afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:25.455261935Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:25.455306064Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: adff3617-5320-4e42-8b10-1430a1eb0afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, dataset_name: DatasetName { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:25.473730197Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:25.473782661Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, dataset_name: DatasetName { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:25.494535482Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/6d2aa15e-618e-402e-b368-cf27fd8def68/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:25.494589325Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/75c788e1-45ef-4fcf-958d-db07331bef52/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:25.494815652Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:25.495024346Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:25.593465716Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:25.59352537Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:25.593532336Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:25.593537567Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:25.593543193Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:25.593548518Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:25.593552446Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:25.593557743Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:25.593561812Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:25.593567071Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:25.593570671Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:25.593594325Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:25.593600053Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:25.593606747Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:25.593610506Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:25.593615673Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:25.593620839Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:25.593625854Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:25.59363087Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:25.59375077Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"9ed53cac-69e7-4694-aba1-678cc90b837e","remote_addr":"[fd00:1122:3344:101::1]:47495","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480\", TimeNotSynchronized), (\"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046\", TimeNotSynchronized), (\"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9\", TimeNotSynchronized), (\"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465\", TimeNotSynchronized), (\"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3\", TimeNotSynchronized), (\"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa\", TimeNotSynchronized), (\"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876\", TimeNotSynchronized), (\"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2\", TimeNotSynchronized), (\"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b\", TimeNotSynchronized), (\"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095\", TimeNotSynchronized), (\"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7\", TimeNotSynchronized), (\"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d\", TimeNotSynchronized), (\"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c\", TimeNotSynchronized), (\"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c\", TimeNotSynchronized), (\"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc\", TimeNotSynchronized), (\"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e\", TimeNotSynchronized), (\"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79\", TimeNotSynchronized), (\"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862\", TimeNotSynchronized), (\"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a\", TimeNotSynchronized)]","latency_us":503451,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:25.593993372Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:472","retry_after":"188.5872ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"9ed53cac-69e7-4694-aba1-678cc90b837e\", \"content-length\": \"124\", \"date\": \"Thu, 25 Jul 2024 05:45:24 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"9ed53cac-69e7-4694-aba1-678cc90b837e\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3f311cd5-ee38-4707-ab8a-9331491f8e7c, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6389ba5c-9d50-4fe4-b1a2-8be3c6861cb0, kind: Service { id: 3f311cd5-ee38-4707-ab8a-9331491f8e7c }, name: Name(\"ntp-3f311cd5-ee38-4707-ab8a-9331491f8e7c\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 235])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4f6a29ee-ca23-4fa8-be69-497060121028, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } } } }, OmicronZoneConfig { id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } } } }, OmicronZoneConfig { id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External } } } }, OmicronZoneConfig { id: b04db43b-dc24-43ce-9d97-32193f88e046, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f04fae05-ec27-43ac-ad4d-ab63c091bf1b, kind: Service { id: b04db43b-dc24-43ce-9d97-32193f88e046 }, name: Name(\"external-dns-b04db43b-dc24-43ce-9d97-32193f88e046\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 154])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 2c3f8453-233f-428f-93cc-0bfefb84929d, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: a6846ef5-8686-4c7e-ae28-d178a4029a85, kind: Service { id: 2c3f8453-233f-428f-93cc-0bfefb84929d }, name: Name(\"external-dns-2c3f8453-233f-428f-93cc-0bfefb84929d\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 164, 190])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: e4efbc31-e513-4290-82f5-c51d8afea0c7, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 735e2401-2ba5-4079-b4a0-7f0184716575, kind: Service { id: e4efbc31-e513-4290-82f5-c51d8afea0c7 }, name: Name(\"nexus-e4efbc31-e513-4290-82f5-c51d8afea0c7\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 154])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 85472347-dfd9-425e-bd98-9b81ebdf3b2b, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: f33554f0-0c2d-4b66-98d0-73365d475c4b, kind: Service { id: 85472347-dfd9-425e-bd98-9b81ebdf3b2b }, name: Name(\"nexus-85472347-dfd9-425e-bd98-9b81ebdf3b2b\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 244])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 221c5e7c-0832-4890-8b0a-f9ba08725876, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 9e8859b7-3d6e-4066-b58c-24fc535c1122, kind: Service { id: 221c5e7c-0832-4890-8b0a-f9ba08725876 }, name: Name(\"nexus-221c5e7c-0832-4890-8b0a-f9ba08725876\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 155])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4b4998f4-10de-49f2-9326-b482137c5fd3, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: a635cc62-7516-4fd9-b490-62d6de386095, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: ac9fa851-e7e1-4295-ac79-d6076526f465, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 1cb96976-2047-44b8-a79c-9fcd74d17bfa, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: a0b9fba0-c330-402f-9bc0-a7510786c98a, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } } } }, OmicronZoneConfig { id: d94d9602-d610-44ce-b9fc-06ded41a1480, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } } } }, OmicronZoneConfig { id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External } } } }, OmicronZoneConfig { id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External } } } }, OmicronZoneConfig { id: dfb1f626-0f55-4385-87de-de59052d98d2, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External } } } }, OmicronZoneConfig { id: adff3617-5320-4e42-8b10-1430a1eb0afc, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:25.783837543Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:434"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:25.784163106Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:25.78416965Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:25.80817455Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:25.808223376Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:25.830135147Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:25.83017861Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4f6a29ee-ca23-4fa8-be69-497060121028, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:25.849572506Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:25.849614226Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4f6a29ee-ca23-4fa8-be69-497060121028, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:25.869843204Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:25.869886039Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:25.892879179Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:25.892933341Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:25.913510989Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:25.913559149Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:25.932383809Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:25.932432063Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b04db43b-dc24-43ce-9d97-32193f88e046, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:25.951940121Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:25.951991113Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: b04db43b-dc24-43ce-9d97-32193f88e046, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2c3f8453-233f-428f-93cc-0bfefb84929d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:25.970746143Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:25.970794781Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2c3f8453-233f-428f-93cc-0bfefb84929d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:25.9908406Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:25.990883098Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a0b9fba0-c330-402f-9bc0-a7510786c98a, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:26.010333012Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:26.010450008Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: a0b9fba0-c330-402f-9bc0-a7510786c98a, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:26.028638785Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:26.028687517Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d94d9602-d610-44ce-b9fc-06ded41a1480, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:26.047758192Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:26.04781119Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d94d9602-d610-44ce-b9fc-06ded41a1480, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:26.067293345Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:26.067340061Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:26.088935196Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:26.088985757Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, dataset_name: DatasetName { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:26.107457157Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:26.107522109Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, dataset_name: DatasetName { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dfb1f626-0f55-4385-87de-de59052d98d2, dataset_name: DatasetName { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:26.126253436Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:26.126299618Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: dfb1f626-0f55-4385-87de-de59052d98d2, dataset_name: DatasetName { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: adff3617-5320-4e42-8b10-1430a1eb0afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:26.144418962Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:26.144472795Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: adff3617-5320-4e42-8b10-1430a1eb0afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, dataset_name: DatasetName { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:26.164435863Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:26.164478005Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, dataset_name: DatasetName { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:26.184429433Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/6d2aa15e-618e-402e-b368-cf27fd8def68/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:26.184491056Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/75c788e1-45ef-4fcf-958d-db07331bef52/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:26.184730508Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:26.184940024Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:26.279230022Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:26.279289199Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:26.279297318Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:26.279301893Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:26.279307415Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:26.279314212Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:26.279320034Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:26.279325463Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:26.279330684Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:26.279335906Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:26.279342619Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:26.27934785Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:26.279353259Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:26.279358642Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:26.279365232Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:26.279387713Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:26.279393123Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:26.279398149Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:26.279403389Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:26.279492557Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"4375fe17-7094-4c02-a1be-da1577d7573d","remote_addr":"[fd00:1122:3344:101::1]:47495","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b\", TimeNotSynchronized), (\"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79\", TimeNotSynchronized), (\"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465\", TimeNotSynchronized), (\"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7\", TimeNotSynchronized), (\"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3\", TimeNotSynchronized), (\"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d\", TimeNotSynchronized), (\"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876\", TimeNotSynchronized), (\"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862\", TimeNotSynchronized), (\"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa\", TimeNotSynchronized), (\"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e\", TimeNotSynchronized), (\"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046\", TimeNotSynchronized), (\"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480\", TimeNotSynchronized), (\"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a\", TimeNotSynchronized), (\"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c\", TimeNotSynchronized), (\"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c\", TimeNotSynchronized), (\"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2\", TimeNotSynchronized), (\"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9\", TimeNotSynchronized), (\"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc\", TimeNotSynchronized), (\"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095\", TimeNotSynchronized)]","latency_us":494199,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:26.279682133Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:472","retry_after":"270.851555ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"4375fe17-7094-4c02-a1be-da1577d7573d\", \"content-length\": \"124\", \"date\": \"Thu, 25 Jul 2024 05:45:25 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"4375fe17-7094-4c02-a1be-da1577d7573d\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3f311cd5-ee38-4707-ab8a-9331491f8e7c, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6389ba5c-9d50-4fe4-b1a2-8be3c6861cb0, kind: Service { id: 3f311cd5-ee38-4707-ab8a-9331491f8e7c }, name: Name(\"ntp-3f311cd5-ee38-4707-ab8a-9331491f8e7c\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 235])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4f6a29ee-ca23-4fa8-be69-497060121028, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } } } }, OmicronZoneConfig { id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } } } }, OmicronZoneConfig { id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External } } } }, OmicronZoneConfig { id: b04db43b-dc24-43ce-9d97-32193f88e046, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f04fae05-ec27-43ac-ad4d-ab63c091bf1b, kind: Service { id: b04db43b-dc24-43ce-9d97-32193f88e046 }, name: Name(\"external-dns-b04db43b-dc24-43ce-9d97-32193f88e046\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 154])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 2c3f8453-233f-428f-93cc-0bfefb84929d, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: a6846ef5-8686-4c7e-ae28-d178a4029a85, kind: Service { id: 2c3f8453-233f-428f-93cc-0bfefb84929d }, name: Name(\"external-dns-2c3f8453-233f-428f-93cc-0bfefb84929d\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 164, 190])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: e4efbc31-e513-4290-82f5-c51d8afea0c7, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 735e2401-2ba5-4079-b4a0-7f0184716575, kind: Service { id: e4efbc31-e513-4290-82f5-c51d8afea0c7 }, name: Name(\"nexus-e4efbc31-e513-4290-82f5-c51d8afea0c7\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 154])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 85472347-dfd9-425e-bd98-9b81ebdf3b2b, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: f33554f0-0c2d-4b66-98d0-73365d475c4b, kind: Service { id: 85472347-dfd9-425e-bd98-9b81ebdf3b2b }, name: Name(\"nexus-85472347-dfd9-425e-bd98-9b81ebdf3b2b\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 244])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 221c5e7c-0832-4890-8b0a-f9ba08725876, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 9e8859b7-3d6e-4066-b58c-24fc535c1122, kind: Service { id: 221c5e7c-0832-4890-8b0a-f9ba08725876 }, name: Name(\"nexus-221c5e7c-0832-4890-8b0a-f9ba08725876\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 155])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4b4998f4-10de-49f2-9326-b482137c5fd3, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: a635cc62-7516-4fd9-b490-62d6de386095, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: ac9fa851-e7e1-4295-ac79-d6076526f465, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 1cb96976-2047-44b8-a79c-9fcd74d17bfa, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: a0b9fba0-c330-402f-9bc0-a7510786c98a, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } } } }, OmicronZoneConfig { id: d94d9602-d610-44ce-b9fc-06ded41a1480, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } } } }, OmicronZoneConfig { id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External } } } }, OmicronZoneConfig { id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External } } } }, OmicronZoneConfig { id: dfb1f626-0f55-4385-87de-de59052d98d2, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External } } } }, OmicronZoneConfig { id: adff3617-5320-4e42-8b10-1430a1eb0afc, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:26.551967559Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:434"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:26.552888653Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:26.5528965Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:26.577278846Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:26.577339955Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:26.598028607Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:26.598076786Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4f6a29ee-ca23-4fa8-be69-497060121028, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:26.618431432Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:26.618484992Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4f6a29ee-ca23-4fa8-be69-497060121028, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:26.638207694Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:26.638257101Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:26.658450174Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:26.658496432Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:26.679328993Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:26.679374003Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:26.699043869Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:26.699090774Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b04db43b-dc24-43ce-9d97-32193f88e046, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:26.718778064Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:26.718837981Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: b04db43b-dc24-43ce-9d97-32193f88e046, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2c3f8453-233f-428f-93cc-0bfefb84929d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:26.738097942Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:26.738144433Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2c3f8453-233f-428f-93cc-0bfefb84929d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:26.757823273Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:26.757866727Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a0b9fba0-c330-402f-9bc0-a7510786c98a, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:26.777543777Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:26.777590888Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: a0b9fba0-c330-402f-9bc0-a7510786c98a, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:26.798904474Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:26.7989539Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d94d9602-d610-44ce-b9fc-06ded41a1480, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:26.819231211Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:26.819279746Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d94d9602-d610-44ce-b9fc-06ded41a1480, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:26.83926492Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:26.839309781Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:26.859394332Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:26.859441546Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, dataset_name: DatasetName { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:26.878439795Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:26.878485321Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, dataset_name: DatasetName { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dfb1f626-0f55-4385-87de-de59052d98d2, dataset_name: DatasetName { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:26.898923913Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:26.898978459Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: dfb1f626-0f55-4385-87de-de59052d98d2, dataset_name: DatasetName { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: adff3617-5320-4e42-8b10-1430a1eb0afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:26.919095437Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:26.919137522Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: adff3617-5320-4e42-8b10-1430a1eb0afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, dataset_name: DatasetName { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:26.939844396Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:26.939959692Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, dataset_name: DatasetName { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:26.959668172Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/6d2aa15e-618e-402e-b368-cf27fd8def68/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:26.959717579Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/75c788e1-45ef-4fcf-958d-db07331bef52/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:26.960002301Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:26.960113697Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:27.061614926Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:27.061683469Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:27.061692207Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:27.061698132Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:27.061712223Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:27.061719479Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:27.06172502Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:27.061756371Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:27.061782481Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:27.061787365Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:27.061791565Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:27.061795465Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:27.061800397Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:27.061804456Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:27.061808422Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:27.061813241Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:27.061817404Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:27.061820563Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:27.061824445Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:27.061935325Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"f9f0da75-d24b-4ff0-b736-44da53249c06","remote_addr":"[fd00:1122:3344:101::1]:47495","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b\", TimeNotSynchronized), (\"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d\", TimeNotSynchronized), (\"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876\", TimeNotSynchronized), (\"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa\", TimeNotSynchronized), (\"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c\", TimeNotSynchronized), (\"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e\", TimeNotSynchronized), (\"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7\", TimeNotSynchronized), (\"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095\", TimeNotSynchronized), (\"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79\", TimeNotSynchronized), (\"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3\", TimeNotSynchronized), (\"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046\", TimeNotSynchronized), (\"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465\", TimeNotSynchronized), (\"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862\", TimeNotSynchronized), (\"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480\", TimeNotSynchronized), (\"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c\", TimeNotSynchronized), (\"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc\", TimeNotSynchronized), (\"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9\", TimeNotSynchronized), (\"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a\", TimeNotSynchronized), (\"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2\", TimeNotSynchronized)]","latency_us":508497,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:27.062144411Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:472","retry_after":"279.317867ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f9f0da75-d24b-4ff0-b736-44da53249c06\", \"content-length\": \"124\", \"date\": \"Thu, 25 Jul 2024 05:45:26 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f9f0da75-d24b-4ff0-b736-44da53249c06\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3f311cd5-ee38-4707-ab8a-9331491f8e7c, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6389ba5c-9d50-4fe4-b1a2-8be3c6861cb0, kind: Service { id: 3f311cd5-ee38-4707-ab8a-9331491f8e7c }, name: Name(\"ntp-3f311cd5-ee38-4707-ab8a-9331491f8e7c\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 235])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4f6a29ee-ca23-4fa8-be69-497060121028, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } } } }, OmicronZoneConfig { id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } } } }, OmicronZoneConfig { id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External } } } }, OmicronZoneConfig { id: b04db43b-dc24-43ce-9d97-32193f88e046, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f04fae05-ec27-43ac-ad4d-ab63c091bf1b, kind: Service { id: b04db43b-dc24-43ce-9d97-32193f88e046 }, name: Name(\"external-dns-b04db43b-dc24-43ce-9d97-32193f88e046\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 154])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 2c3f8453-233f-428f-93cc-0bfefb84929d, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: a6846ef5-8686-4c7e-ae28-d178a4029a85, kind: Service { id: 2c3f8453-233f-428f-93cc-0bfefb84929d }, name: Name(\"external-dns-2c3f8453-233f-428f-93cc-0bfefb84929d\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 164, 190])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: e4efbc31-e513-4290-82f5-c51d8afea0c7, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 735e2401-2ba5-4079-b4a0-7f0184716575, kind: Service { id: e4efbc31-e513-4290-82f5-c51d8afea0c7 }, name: Name(\"nexus-e4efbc31-e513-4290-82f5-c51d8afea0c7\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 154])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 85472347-dfd9-425e-bd98-9b81ebdf3b2b, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: f33554f0-0c2d-4b66-98d0-73365d475c4b, kind: Service { id: 85472347-dfd9-425e-bd98-9b81ebdf3b2b }, name: Name(\"nexus-85472347-dfd9-425e-bd98-9b81ebdf3b2b\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 244])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 221c5e7c-0832-4890-8b0a-f9ba08725876, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 9e8859b7-3d6e-4066-b58c-24fc535c1122, kind: Service { id: 221c5e7c-0832-4890-8b0a-f9ba08725876 }, name: Name(\"nexus-221c5e7c-0832-4890-8b0a-f9ba08725876\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 155])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4b4998f4-10de-49f2-9326-b482137c5fd3, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: a635cc62-7516-4fd9-b490-62d6de386095, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: ac9fa851-e7e1-4295-ac79-d6076526f465, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 1cb96976-2047-44b8-a79c-9fcd74d17bfa, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: a0b9fba0-c330-402f-9bc0-a7510786c98a, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } } } }, OmicronZoneConfig { id: d94d9602-d610-44ce-b9fc-06ded41a1480, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } } } }, OmicronZoneConfig { id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External } } } }, OmicronZoneConfig { id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External } } } }, OmicronZoneConfig { id: dfb1f626-0f55-4385-87de-de59052d98d2, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External } } } }, OmicronZoneConfig { id: adff3617-5320-4e42-8b10-1430a1eb0afc, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:27.343237185Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:434"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:27.344100536Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:27.344110033Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:27.367823833Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:27.367875267Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:27.387133947Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:27.387174946Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4f6a29ee-ca23-4fa8-be69-497060121028, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:27.408032862Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:27.408078876Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4f6a29ee-ca23-4fa8-be69-497060121028, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:27.427983148Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:27.428028983Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:27.448463525Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:27.448504569Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:27.467535972Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:27.467581132Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:27.486190059Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:27.486229848Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b04db43b-dc24-43ce-9d97-32193f88e046, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:27.504257868Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:27.504299981Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: b04db43b-dc24-43ce-9d97-32193f88e046, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2c3f8453-233f-428f-93cc-0bfefb84929d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:27.524022639Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:27.524106397Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2c3f8453-233f-428f-93cc-0bfefb84929d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:27.543120599Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:27.543161794Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a0b9fba0-c330-402f-9bc0-a7510786c98a, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:27.564036892Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:27.564079774Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: a0b9fba0-c330-402f-9bc0-a7510786c98a, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:27.582485834Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:27.582531181Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d94d9602-d610-44ce-b9fc-06ded41a1480, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:27.602514243Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:27.602558326Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d94d9602-d610-44ce-b9fc-06ded41a1480, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:27.620959599Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:27.621000503Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:27.641448205Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:27.641494529Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, dataset_name: DatasetName { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:27.662086109Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:27.662131055Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, dataset_name: DatasetName { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dfb1f626-0f55-4385-87de-de59052d98d2, dataset_name: DatasetName { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:27.681760338Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:27.681801926Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: dfb1f626-0f55-4385-87de-de59052d98d2, dataset_name: DatasetName { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: adff3617-5320-4e42-8b10-1430a1eb0afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:27.702519427Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:27.702567606Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: adff3617-5320-4e42-8b10-1430a1eb0afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, dataset_name: DatasetName { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:27.723163015Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:27.723217428Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, dataset_name: DatasetName { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:27.743632288Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/6d2aa15e-618e-402e-b368-cf27fd8def68/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:27.743692299Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/75c788e1-45ef-4fcf-958d-db07331bef52/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:27.743915157Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:27.744123054Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:27.843996486Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:27.844043418Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:27.844048331Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:27.844054022Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:27.844059207Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:27.844064663Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:27.844069763Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:27.844084857Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:27.844090069Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:27.844094992Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:27.844101086Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:27.84410613Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:27.844111155Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:27.844116114Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:27.844122189Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:27.844127045Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:27.844132212Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:27.844138202Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:27.84414304Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:27.844249702Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"f9684fbf-a16a-481f-b7e8-91821c9486d8","remote_addr":"[fd00:1122:3344:101::1]:47495","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b\", TimeNotSynchronized), (\"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9\", TimeNotSynchronized), (\"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095\", TimeNotSynchronized), (\"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2\", TimeNotSynchronized), (\"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862\", TimeNotSynchronized), (\"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e\", TimeNotSynchronized), (\"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480\", TimeNotSynchronized), (\"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c\", TimeNotSynchronized), (\"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79\", TimeNotSynchronized), (\"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7\", TimeNotSynchronized), (\"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3\", TimeNotSynchronized), (\"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d\", TimeNotSynchronized), (\"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa\", TimeNotSynchronized), (\"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a\", TimeNotSynchronized), (\"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c\", TimeNotSynchronized), (\"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046\", TimeNotSynchronized), (\"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc\", TimeNotSynchronized), (\"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465\", TimeNotSynchronized), (\"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876\", TimeNotSynchronized)]","latency_us":499554,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:27.844499762Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:472","retry_after":"325.924107ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f9684fbf-a16a-481f-b7e8-91821c9486d8\", \"content-length\": \"124\", \"date\": \"Thu, 25 Jul 2024 05:45:27 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f9684fbf-a16a-481f-b7e8-91821c9486d8\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3f311cd5-ee38-4707-ab8a-9331491f8e7c, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6389ba5c-9d50-4fe4-b1a2-8be3c6861cb0, kind: Service { id: 3f311cd5-ee38-4707-ab8a-9331491f8e7c }, name: Name(\"ntp-3f311cd5-ee38-4707-ab8a-9331491f8e7c\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 235])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4f6a29ee-ca23-4fa8-be69-497060121028, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } } } }, OmicronZoneConfig { id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } } } }, OmicronZoneConfig { id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External } } } }, OmicronZoneConfig { id: b04db43b-dc24-43ce-9d97-32193f88e046, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f04fae05-ec27-43ac-ad4d-ab63c091bf1b, kind: Service { id: b04db43b-dc24-43ce-9d97-32193f88e046 }, name: Name(\"external-dns-b04db43b-dc24-43ce-9d97-32193f88e046\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 154])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 2c3f8453-233f-428f-93cc-0bfefb84929d, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: a6846ef5-8686-4c7e-ae28-d178a4029a85, kind: Service { id: 2c3f8453-233f-428f-93cc-0bfefb84929d }, name: Name(\"external-dns-2c3f8453-233f-428f-93cc-0bfefb84929d\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 164, 190])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: e4efbc31-e513-4290-82f5-c51d8afea0c7, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 735e2401-2ba5-4079-b4a0-7f0184716575, kind: Service { id: e4efbc31-e513-4290-82f5-c51d8afea0c7 }, name: Name(\"nexus-e4efbc31-e513-4290-82f5-c51d8afea0c7\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 154])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 85472347-dfd9-425e-bd98-9b81ebdf3b2b, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: f33554f0-0c2d-4b66-98d0-73365d475c4b, kind: Service { id: 85472347-dfd9-425e-bd98-9b81ebdf3b2b }, name: Name(\"nexus-85472347-dfd9-425e-bd98-9b81ebdf3b2b\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 244])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 221c5e7c-0832-4890-8b0a-f9ba08725876, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 9e8859b7-3d6e-4066-b58c-24fc535c1122, kind: Service { id: 221c5e7c-0832-4890-8b0a-f9ba08725876 }, name: Name(\"nexus-221c5e7c-0832-4890-8b0a-f9ba08725876\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 155])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4b4998f4-10de-49f2-9326-b482137c5fd3, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: a635cc62-7516-4fd9-b490-62d6de386095, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: ac9fa851-e7e1-4295-ac79-d6076526f465, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 1cb96976-2047-44b8-a79c-9fcd74d17bfa, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: a0b9fba0-c330-402f-9bc0-a7510786c98a, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } } } }, OmicronZoneConfig { id: d94d9602-d610-44ce-b9fc-06ded41a1480, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } } } }, OmicronZoneConfig { id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External } } } }, OmicronZoneConfig { id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External } } } }, OmicronZoneConfig { id: dfb1f626-0f55-4385-87de-de59052d98d2, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External } } } }, OmicronZoneConfig { id: adff3617-5320-4e42-8b10-1430a1eb0afc, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:28.172901327Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:434"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:28.17329669Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:28.173303299Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:28.197238231Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:28.197366475Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:28.218243873Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:28.218284655Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4f6a29ee-ca23-4fa8-be69-497060121028, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:28.239052459Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:28.239096129Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4f6a29ee-ca23-4fa8-be69-497060121028, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:28.25882085Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:28.258862448Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:28.278205508Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:28.278242745Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:28.296102057Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:28.29614897Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:28.315788818Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:28.315828166Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b04db43b-dc24-43ce-9d97-32193f88e046, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:28.335281725Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:28.335321991Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: b04db43b-dc24-43ce-9d97-32193f88e046, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2c3f8453-233f-428f-93cc-0bfefb84929d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:28.356151353Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:28.35620646Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2c3f8453-233f-428f-93cc-0bfefb84929d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:28.376914719Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:28.376969967Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a0b9fba0-c330-402f-9bc0-a7510786c98a, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:28.398617786Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:28.398662262Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: a0b9fba0-c330-402f-9bc0-a7510786c98a, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:28.417525211Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:28.417568196Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d94d9602-d610-44ce-b9fc-06ded41a1480, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:28.436312533Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:28.436361077Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d94d9602-d610-44ce-b9fc-06ded41a1480, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:28.455393978Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:28.455437939Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:28.475988588Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:28.47611107Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, dataset_name: DatasetName { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:28.496284271Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:28.496331692Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, dataset_name: DatasetName { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dfb1f626-0f55-4385-87de-de59052d98d2, dataset_name: DatasetName { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:28.518496577Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:28.518543725Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: dfb1f626-0f55-4385-87de-de59052d98d2, dataset_name: DatasetName { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: adff3617-5320-4e42-8b10-1430a1eb0afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:28.539550511Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:28.539598221Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: adff3617-5320-4e42-8b10-1430a1eb0afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, dataset_name: DatasetName { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:28.560376714Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:28.560424865Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, dataset_name: DatasetName { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:28.582055124Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/6d2aa15e-618e-402e-b368-cf27fd8def68/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:28.582120619Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/75c788e1-45ef-4fcf-958d-db07331bef52/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:28.58244345Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:28.58260385Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:28.678499868Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:28.678550344Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:28.678556831Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:28.678629169Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:28.678634223Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:28.678643664Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:28.678648717Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:28.678653555Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:28.678660014Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:28.67866473Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:28.678669792Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:28.67868484Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:28.678691084Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:28.678695846Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:28.678700449Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:28.678705174Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:28.678709562Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:28.678714352Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:28.678719332Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:28.678803183Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"3bde4b7e-0be2-4386-a176-a3150fa8f837","remote_addr":"[fd00:1122:3344:101::1]:47495","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3\", TimeNotSynchronized), (\"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c\", TimeNotSynchronized), (\"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d\", TimeNotSynchronized), (\"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c\", TimeNotSynchronized), (\"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862\", TimeNotSynchronized), (\"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa\", TimeNotSynchronized), (\"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046\", TimeNotSynchronized), (\"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7\", TimeNotSynchronized), (\"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480\", TimeNotSynchronized), (\"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876\", TimeNotSynchronized), (\"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465\", TimeNotSynchronized), (\"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b\", TimeNotSynchronized), (\"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095\", TimeNotSynchronized), (\"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a\", TimeNotSynchronized), (\"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e\", TimeNotSynchronized), (\"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2\", TimeNotSynchronized), (\"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc\", TimeNotSynchronized), (\"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9\", TimeNotSynchronized), (\"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79\", TimeNotSynchronized)]","latency_us":504354,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:28.678997457Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:472","retry_after":"751.935821ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3bde4b7e-0be2-4386-a176-a3150fa8f837\", \"content-length\": \"124\", \"date\": \"Thu, 25 Jul 2024 05:45:27 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"3bde4b7e-0be2-4386-a176-a3150fa8f837\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3f311cd5-ee38-4707-ab8a-9331491f8e7c, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6389ba5c-9d50-4fe4-b1a2-8be3c6861cb0, kind: Service { id: 3f311cd5-ee38-4707-ab8a-9331491f8e7c }, name: Name(\"ntp-3f311cd5-ee38-4707-ab8a-9331491f8e7c\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 235])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4f6a29ee-ca23-4fa8-be69-497060121028, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } } } }, OmicronZoneConfig { id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } } } }, OmicronZoneConfig { id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External } } } }, OmicronZoneConfig { id: b04db43b-dc24-43ce-9d97-32193f88e046, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f04fae05-ec27-43ac-ad4d-ab63c091bf1b, kind: Service { id: b04db43b-dc24-43ce-9d97-32193f88e046 }, name: Name(\"external-dns-b04db43b-dc24-43ce-9d97-32193f88e046\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 154])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 2c3f8453-233f-428f-93cc-0bfefb84929d, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: a6846ef5-8686-4c7e-ae28-d178a4029a85, kind: Service { id: 2c3f8453-233f-428f-93cc-0bfefb84929d }, name: Name(\"external-dns-2c3f8453-233f-428f-93cc-0bfefb84929d\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 164, 190])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: e4efbc31-e513-4290-82f5-c51d8afea0c7, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 735e2401-2ba5-4079-b4a0-7f0184716575, kind: Service { id: e4efbc31-e513-4290-82f5-c51d8afea0c7 }, name: Name(\"nexus-e4efbc31-e513-4290-82f5-c51d8afea0c7\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 154])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 85472347-dfd9-425e-bd98-9b81ebdf3b2b, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: f33554f0-0c2d-4b66-98d0-73365d475c4b, kind: Service { id: 85472347-dfd9-425e-bd98-9b81ebdf3b2b }, name: Name(\"nexus-85472347-dfd9-425e-bd98-9b81ebdf3b2b\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 244])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 221c5e7c-0832-4890-8b0a-f9ba08725876, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 9e8859b7-3d6e-4066-b58c-24fc535c1122, kind: Service { id: 221c5e7c-0832-4890-8b0a-f9ba08725876 }, name: Name(\"nexus-221c5e7c-0832-4890-8b0a-f9ba08725876\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 155])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4b4998f4-10de-49f2-9326-b482137c5fd3, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: a635cc62-7516-4fd9-b490-62d6de386095, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: ac9fa851-e7e1-4295-ac79-d6076526f465, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 1cb96976-2047-44b8-a79c-9fcd74d17bfa, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: a0b9fba0-c330-402f-9bc0-a7510786c98a, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } } } }, OmicronZoneConfig { id: d94d9602-d610-44ce-b9fc-06ded41a1480, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } } } }, OmicronZoneConfig { id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External } } } }, OmicronZoneConfig { id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External } } } }, OmicronZoneConfig { id: dfb1f626-0f55-4385-87de-de59052d98d2, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External } } } }, OmicronZoneConfig { id: adff3617-5320-4e42-8b10-1430a1eb0afc, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:29.433781924Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:434"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:29.434773193Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:29.434781133Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:29.458341421Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:29.458392545Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:29.47886457Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:29.478927581Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4f6a29ee-ca23-4fa8-be69-497060121028, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:29.497443104Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:29.497491059Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4f6a29ee-ca23-4fa8-be69-497060121028, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:29.516153263Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:29.516195873Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:29.538643529Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:29.538692139Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:29.558057299Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:29.558101335Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:29.576660395Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:29.576705021Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b04db43b-dc24-43ce-9d97-32193f88e046, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:29.596874587Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:29.596920423Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: b04db43b-dc24-43ce-9d97-32193f88e046, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2c3f8453-233f-428f-93cc-0bfefb84929d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:29.616874089Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:29.616911093Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2c3f8453-233f-428f-93cc-0bfefb84929d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:29.637550174Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:29.637591875Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a0b9fba0-c330-402f-9bc0-a7510786c98a, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:29.657274274Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:29.657321262Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: a0b9fba0-c330-402f-9bc0-a7510786c98a, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:29.676141409Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:29.676186672Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d94d9602-d610-44ce-b9fc-06ded41a1480, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:29.69483325Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:29.694889754Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d94d9602-d610-44ce-b9fc-06ded41a1480, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:29.71532384Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:29.715445411Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:29.734697224Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:29.734743726Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, dataset_name: DatasetName { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:29.754320651Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:29.754379912Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, dataset_name: DatasetName { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dfb1f626-0f55-4385-87de-de59052d98d2, dataset_name: DatasetName { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:29.773967025Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:29.774007207Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: dfb1f626-0f55-4385-87de-de59052d98d2, dataset_name: DatasetName { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: adff3617-5320-4e42-8b10-1430a1eb0afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:29.794145349Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:29.794195618Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: adff3617-5320-4e42-8b10-1430a1eb0afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, dataset_name: DatasetName { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:29.813355562Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:29.813397723Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, dataset_name: DatasetName { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:29.834337798Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/6d2aa15e-618e-402e-b368-cf27fd8def68/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:29.83439029Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/75c788e1-45ef-4fcf-958d-db07331bef52/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:29.834627117Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:29.834855022Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:29.931818642Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:29.931886472Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:29.931898266Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:29.931906441Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:29.931913472Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:29.931921705Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:29.931931342Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:29.931941064Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:29.932016009Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:29.93202618Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:29.932032144Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:29.932038322Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:29.932043113Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:29.932048063Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:29.9320529Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:29.932068773Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:29.93207481Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:29.932080829Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:29.932084326Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:29.932157987Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"8ca8134e-3eed-487a-93eb-ed5cb8048079","remote_addr":"[fd00:1122:3344:101::1]:47495","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7\", TimeNotSynchronized), (\"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876\", TimeNotSynchronized), (\"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046\", TimeNotSynchronized), (\"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862\", TimeNotSynchronized), (\"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480\", TimeNotSynchronized), (\"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a\", TimeNotSynchronized), (\"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c\", TimeNotSynchronized), (\"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9\", TimeNotSynchronized), (\"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b\", TimeNotSynchronized), (\"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095\", TimeNotSynchronized), (\"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79\", TimeNotSynchronized), (\"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e\", TimeNotSynchronized), (\"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d\", TimeNotSynchronized), (\"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc\", TimeNotSynchronized), (\"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c\", TimeNotSynchronized), (\"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa\", TimeNotSynchronized), (\"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2\", TimeNotSynchronized), (\"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3\", TimeNotSynchronized), (\"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465\", TimeNotSynchronized)]","latency_us":496875,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:29.93236308Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:472","retry_after":"449.825485ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"8ca8134e-3eed-487a-93eb-ed5cb8048079\", \"content-length\": \"124\", \"date\": \"Thu, 25 Jul 2024 05:45:29 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"8ca8134e-3eed-487a-93eb-ed5cb8048079\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3f311cd5-ee38-4707-ab8a-9331491f8e7c, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6389ba5c-9d50-4fe4-b1a2-8be3c6861cb0, kind: Service { id: 3f311cd5-ee38-4707-ab8a-9331491f8e7c }, name: Name(\"ntp-3f311cd5-ee38-4707-ab8a-9331491f8e7c\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 235])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4f6a29ee-ca23-4fa8-be69-497060121028, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } } } }, OmicronZoneConfig { id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } } } }, OmicronZoneConfig { id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External } } } }, OmicronZoneConfig { id: b04db43b-dc24-43ce-9d97-32193f88e046, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f04fae05-ec27-43ac-ad4d-ab63c091bf1b, kind: Service { id: b04db43b-dc24-43ce-9d97-32193f88e046 }, name: Name(\"external-dns-b04db43b-dc24-43ce-9d97-32193f88e046\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 154])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 2c3f8453-233f-428f-93cc-0bfefb84929d, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: a6846ef5-8686-4c7e-ae28-d178a4029a85, kind: Service { id: 2c3f8453-233f-428f-93cc-0bfefb84929d }, name: Name(\"external-dns-2c3f8453-233f-428f-93cc-0bfefb84929d\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 164, 190])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: e4efbc31-e513-4290-82f5-c51d8afea0c7, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 735e2401-2ba5-4079-b4a0-7f0184716575, kind: Service { id: e4efbc31-e513-4290-82f5-c51d8afea0c7 }, name: Name(\"nexus-e4efbc31-e513-4290-82f5-c51d8afea0c7\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 154])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 85472347-dfd9-425e-bd98-9b81ebdf3b2b, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: f33554f0-0c2d-4b66-98d0-73365d475c4b, kind: Service { id: 85472347-dfd9-425e-bd98-9b81ebdf3b2b }, name: Name(\"nexus-85472347-dfd9-425e-bd98-9b81ebdf3b2b\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 244])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 221c5e7c-0832-4890-8b0a-f9ba08725876, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 9e8859b7-3d6e-4066-b58c-24fc535c1122, kind: Service { id: 221c5e7c-0832-4890-8b0a-f9ba08725876 }, name: Name(\"nexus-221c5e7c-0832-4890-8b0a-f9ba08725876\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 155])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4b4998f4-10de-49f2-9326-b482137c5fd3, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: a635cc62-7516-4fd9-b490-62d6de386095, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: ac9fa851-e7e1-4295-ac79-d6076526f465, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 1cb96976-2047-44b8-a79c-9fcd74d17bfa, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: a0b9fba0-c330-402f-9bc0-a7510786c98a, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } } } }, OmicronZoneConfig { id: d94d9602-d610-44ce-b9fc-06ded41a1480, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } } } }, OmicronZoneConfig { id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External } } } }, OmicronZoneConfig { id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External } } } }, OmicronZoneConfig { id: dfb1f626-0f55-4385-87de-de59052d98d2, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External } } } }, OmicronZoneConfig { id: adff3617-5320-4e42-8b10-1430a1eb0afc, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:30.384152119Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:434"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:30.38459076Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:30.38459872Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:30.408172739Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:30.408225672Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:30.430724859Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:30.430769354Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4f6a29ee-ca23-4fa8-be69-497060121028, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:30.451073531Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:30.451130552Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4f6a29ee-ca23-4fa8-be69-497060121028, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:30.471509495Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:30.471556061Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:30.491885458Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:30.49193298Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:30.510959856Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:30.511004566Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:30.531786329Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:30.531825676Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b04db43b-dc24-43ce-9d97-32193f88e046, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:30.550243704Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:30.550296112Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: b04db43b-dc24-43ce-9d97-32193f88e046, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2c3f8453-233f-428f-93cc-0bfefb84929d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:30.570386852Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:30.570434769Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2c3f8453-233f-428f-93cc-0bfefb84929d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:30.59110985Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:30.591154297Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a0b9fba0-c330-402f-9bc0-a7510786c98a, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:30.610902665Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:30.610948331Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: a0b9fba0-c330-402f-9bc0-a7510786c98a, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:30.630549485Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:30.630593605Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d94d9602-d610-44ce-b9fc-06ded41a1480, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:30.65009338Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:30.650135185Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d94d9602-d610-44ce-b9fc-06ded41a1480, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:30.670762562Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:30.670802191Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:30.69251427Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:30.692556215Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, dataset_name: DatasetName { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:30.712853744Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:30.712892369Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, dataset_name: DatasetName { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dfb1f626-0f55-4385-87de-de59052d98d2, dataset_name: DatasetName { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:30.732825706Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:30.732877888Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: dfb1f626-0f55-4385-87de-de59052d98d2, dataset_name: DatasetName { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: adff3617-5320-4e42-8b10-1430a1eb0afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:30.752460616Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:30.75250033Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: adff3617-5320-4e42-8b10-1430a1eb0afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, dataset_name: DatasetName { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:30.771124146Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:30.771164497Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, dataset_name: DatasetName { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:30.79179288Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/6d2aa15e-618e-402e-b368-cf27fd8def68/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:30.791837815Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/75c788e1-45ef-4fcf-958d-db07331bef52/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:30.792054073Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:30.79216337Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:30.891642387Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:30.891703354Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:30.891709691Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:30.891715298Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:30.891737545Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:30.891743133Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:30.89174857Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:30.891755612Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:30.891761555Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:30.891768484Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:30.89177393Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:30.891778984Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:30.891785574Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:30.89179134Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:30.891796796Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:30.891805075Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:30.891829131Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:30.89184716Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:30.89186081Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:30.891993555Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"aa716d32-6c05-4329-9828-4353f98bf632","remote_addr":"[fd00:1122:3344:101::1]:47495","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79\", TimeNotSynchronized), (\"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9\", TimeNotSynchronized), (\"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046\", TimeNotSynchronized), (\"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a\", TimeNotSynchronized), (\"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480\", TimeNotSynchronized), (\"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876\", TimeNotSynchronized), (\"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2\", TimeNotSynchronized), (\"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa\", TimeNotSynchronized), (\"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc\", TimeNotSynchronized), (\"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465\", TimeNotSynchronized), (\"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c\", TimeNotSynchronized), (\"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7\", TimeNotSynchronized), (\"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862\", TimeNotSynchronized), (\"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c\", TimeNotSynchronized), (\"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b\", TimeNotSynchronized), (\"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3\", TimeNotSynchronized), (\"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095\", TimeNotSynchronized), (\"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e\", TimeNotSynchronized), (\"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d\", TimeNotSynchronized)]","latency_us":506272,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:30.892249967Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:472","retry_after":"815.289752ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"aa716d32-6c05-4329-9828-4353f98bf632\", \"content-length\": \"124\", \"date\": \"Thu, 25 Jul 2024 05:45:30 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"aa716d32-6c05-4329-9828-4353f98bf632\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3f311cd5-ee38-4707-ab8a-9331491f8e7c, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6389ba5c-9d50-4fe4-b1a2-8be3c6861cb0, kind: Service { id: 3f311cd5-ee38-4707-ab8a-9331491f8e7c }, name: Name(\"ntp-3f311cd5-ee38-4707-ab8a-9331491f8e7c\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 235])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4f6a29ee-ca23-4fa8-be69-497060121028, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } } } }, OmicronZoneConfig { id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } } } }, OmicronZoneConfig { id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External } } } }, OmicronZoneConfig { id: b04db43b-dc24-43ce-9d97-32193f88e046, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f04fae05-ec27-43ac-ad4d-ab63c091bf1b, kind: Service { id: b04db43b-dc24-43ce-9d97-32193f88e046 }, name: Name(\"external-dns-b04db43b-dc24-43ce-9d97-32193f88e046\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 154])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 2c3f8453-233f-428f-93cc-0bfefb84929d, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: a6846ef5-8686-4c7e-ae28-d178a4029a85, kind: Service { id: 2c3f8453-233f-428f-93cc-0bfefb84929d }, name: Name(\"external-dns-2c3f8453-233f-428f-93cc-0bfefb84929d\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 164, 190])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: e4efbc31-e513-4290-82f5-c51d8afea0c7, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 735e2401-2ba5-4079-b4a0-7f0184716575, kind: Service { id: e4efbc31-e513-4290-82f5-c51d8afea0c7 }, name: Name(\"nexus-e4efbc31-e513-4290-82f5-c51d8afea0c7\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 154])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 85472347-dfd9-425e-bd98-9b81ebdf3b2b, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: f33554f0-0c2d-4b66-98d0-73365d475c4b, kind: Service { id: 85472347-dfd9-425e-bd98-9b81ebdf3b2b }, name: Name(\"nexus-85472347-dfd9-425e-bd98-9b81ebdf3b2b\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 244])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 221c5e7c-0832-4890-8b0a-f9ba08725876, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 9e8859b7-3d6e-4066-b58c-24fc535c1122, kind: Service { id: 221c5e7c-0832-4890-8b0a-f9ba08725876 }, name: Name(\"nexus-221c5e7c-0832-4890-8b0a-f9ba08725876\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 155])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4b4998f4-10de-49f2-9326-b482137c5fd3, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: a635cc62-7516-4fd9-b490-62d6de386095, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: ac9fa851-e7e1-4295-ac79-d6076526f465, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 1cb96976-2047-44b8-a79c-9fcd74d17bfa, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: a0b9fba0-c330-402f-9bc0-a7510786c98a, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } } } }, OmicronZoneConfig { id: d94d9602-d610-44ce-b9fc-06ded41a1480, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } } } }, OmicronZoneConfig { id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External } } } }, OmicronZoneConfig { id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External } } } }, OmicronZoneConfig { id: dfb1f626-0f55-4385-87de-de59052d98d2, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External } } } }, OmicronZoneConfig { id: adff3617-5320-4e42-8b10-1430a1eb0afc, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:31.710999623Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:434"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:31.711944725Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:31.711957635Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:31.736511086Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:31.736557794Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:31.758163448Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:31.758228512Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4f6a29ee-ca23-4fa8-be69-497060121028, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:31.778661428Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:31.778722507Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4f6a29ee-ca23-4fa8-be69-497060121028, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:31.796653571Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:31.796699341Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:31.816999414Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:31.817117261Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:31.835604715Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:31.835643931Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:31.854946287Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:31.854989619Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b04db43b-dc24-43ce-9d97-32193f88e046, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:31.873809414Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:31.873857228Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: b04db43b-dc24-43ce-9d97-32193f88e046, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2c3f8453-233f-428f-93cc-0bfefb84929d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:31.892189157Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:31.892236924Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2c3f8453-233f-428f-93cc-0bfefb84929d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:31.910583437Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:31.910626404Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a0b9fba0-c330-402f-9bc0-a7510786c98a, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:31.932077162Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:31.932120981Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: a0b9fba0-c330-402f-9bc0-a7510786c98a, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:31.953194313Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:31.953232358Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d94d9602-d610-44ce-b9fc-06ded41a1480, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:31.972286273Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:31.972329727Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d94d9602-d610-44ce-b9fc-06ded41a1480, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:31.992821509Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:31.992870466Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:32.011838485Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:32.01188177Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, dataset_name: DatasetName { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:32.033138629Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:32.033178408Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, dataset_name: DatasetName { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dfb1f626-0f55-4385-87de-de59052d98d2, dataset_name: DatasetName { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:32.054353911Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:32.054402662Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: dfb1f626-0f55-4385-87de-de59052d98d2, dataset_name: DatasetName { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: adff3617-5320-4e42-8b10-1430a1eb0afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:32.072725209Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:32.072839103Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: adff3617-5320-4e42-8b10-1430a1eb0afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, dataset_name: DatasetName { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:32.09288751Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:32.09292773Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, dataset_name: DatasetName { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:32.114160189Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/6d2aa15e-618e-402e-b368-cf27fd8def68/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:32.114215033Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/75c788e1-45ef-4fcf-958d-db07331bef52/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:32.114432126Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:32.114563389Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:32.218660849Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:32.218702662Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:32.218707818Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:32.2187117Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:32.218715094Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:32.218718957Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:32.218722116Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:32.218725163Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:32.218743538Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:32.218747944Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:32.218751872Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:32.218756833Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:32.218760705Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:32.218764586Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:32.21876842Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:32.21877233Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:32.218776258Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:32.218780148Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:32.218784105Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:32.218885845Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"30e8eb38-8981-4f11-a681-134c44b08a3e","remote_addr":"[fd00:1122:3344:101::1]:47495","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876\", TimeNotSynchronized), (\"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c\", TimeNotSynchronized), (\"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b\", TimeNotSynchronized), (\"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3\", TimeNotSynchronized), (\"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c\", TimeNotSynchronized), (\"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa\", TimeNotSynchronized), (\"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480\", TimeNotSynchronized), (\"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e\", TimeNotSynchronized), (\"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862\", TimeNotSynchronized), (\"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095\", TimeNotSynchronized), (\"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465\", TimeNotSynchronized), (\"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046\", TimeNotSynchronized), (\"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d\", TimeNotSynchronized), (\"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7\", TimeNotSynchronized), (\"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a\", TimeNotSynchronized), (\"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc\", TimeNotSynchronized), (\"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9\", TimeNotSynchronized), (\"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79\", TimeNotSynchronized), (\"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2\", TimeNotSynchronized)]","latency_us":506424,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:32.219079321Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:472","retry_after":"1.224533693s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"30e8eb38-8981-4f11-a681-134c44b08a3e\", \"content-length\": \"124\", \"date\": \"Thu, 25 Jul 2024 05:45:32 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"30e8eb38-8981-4f11-a681-134c44b08a3e\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3f311cd5-ee38-4707-ab8a-9331491f8e7c, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6389ba5c-9d50-4fe4-b1a2-8be3c6861cb0, kind: Service { id: 3f311cd5-ee38-4707-ab8a-9331491f8e7c }, name: Name(\"ntp-3f311cd5-ee38-4707-ab8a-9331491f8e7c\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 235])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4f6a29ee-ca23-4fa8-be69-497060121028, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } } } }, OmicronZoneConfig { id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } } } }, OmicronZoneConfig { id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External } } } }, OmicronZoneConfig { id: b04db43b-dc24-43ce-9d97-32193f88e046, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f04fae05-ec27-43ac-ad4d-ab63c091bf1b, kind: Service { id: b04db43b-dc24-43ce-9d97-32193f88e046 }, name: Name(\"external-dns-b04db43b-dc24-43ce-9d97-32193f88e046\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 154])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 2c3f8453-233f-428f-93cc-0bfefb84929d, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: a6846ef5-8686-4c7e-ae28-d178a4029a85, kind: Service { id: 2c3f8453-233f-428f-93cc-0bfefb84929d }, name: Name(\"external-dns-2c3f8453-233f-428f-93cc-0bfefb84929d\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 164, 190])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: e4efbc31-e513-4290-82f5-c51d8afea0c7, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 735e2401-2ba5-4079-b4a0-7f0184716575, kind: Service { id: e4efbc31-e513-4290-82f5-c51d8afea0c7 }, name: Name(\"nexus-e4efbc31-e513-4290-82f5-c51d8afea0c7\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 154])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 85472347-dfd9-425e-bd98-9b81ebdf3b2b, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: f33554f0-0c2d-4b66-98d0-73365d475c4b, kind: Service { id: 85472347-dfd9-425e-bd98-9b81ebdf3b2b }, name: Name(\"nexus-85472347-dfd9-425e-bd98-9b81ebdf3b2b\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 244])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 221c5e7c-0832-4890-8b0a-f9ba08725876, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 9e8859b7-3d6e-4066-b58c-24fc535c1122, kind: Service { id: 221c5e7c-0832-4890-8b0a-f9ba08725876 }, name: Name(\"nexus-221c5e7c-0832-4890-8b0a-f9ba08725876\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 155])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4b4998f4-10de-49f2-9326-b482137c5fd3, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: a635cc62-7516-4fd9-b490-62d6de386095, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: ac9fa851-e7e1-4295-ac79-d6076526f465, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 1cb96976-2047-44b8-a79c-9fcd74d17bfa, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: a0b9fba0-c330-402f-9bc0-a7510786c98a, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } } } }, OmicronZoneConfig { id: d94d9602-d610-44ce-b9fc-06ded41a1480, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } } } }, OmicronZoneConfig { id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External } } } }, OmicronZoneConfig { id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External } } } }, OmicronZoneConfig { id: dfb1f626-0f55-4385-87de-de59052d98d2, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External } } } }, OmicronZoneConfig { id: adff3617-5320-4e42-8b10-1430a1eb0afc, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:33.448074221Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:434"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:33.448428839Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:33.44843949Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:33.471108798Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:33.471161974Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:33.493689244Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:33.493734958Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4f6a29ee-ca23-4fa8-be69-497060121028, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:33.517043694Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:33.517088827Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4f6a29ee-ca23-4fa8-be69-497060121028, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:33.538619685Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:33.538669401Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:33.559692067Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:33.559737565Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:33.579558335Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:33.579602436Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:33.600463488Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:33.60050426Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b04db43b-dc24-43ce-9d97-32193f88e046, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:33.621473515Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:33.621522041Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: b04db43b-dc24-43ce-9d97-32193f88e046, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2c3f8453-233f-428f-93cc-0bfefb84929d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:33.63964748Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:33.639689688Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2c3f8453-233f-428f-93cc-0bfefb84929d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:33.660679691Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:33.660729371Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a0b9fba0-c330-402f-9bc0-a7510786c98a, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:33.680881935Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:33.680923552Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: a0b9fba0-c330-402f-9bc0-a7510786c98a, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:33.701431471Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:33.701467444Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d94d9602-d610-44ce-b9fc-06ded41a1480, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:33.722059945Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:33.722102181Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d94d9602-d610-44ce-b9fc-06ded41a1480, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:33.743823223Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:33.743872041Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:33.762655063Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:33.762713217Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, dataset_name: DatasetName { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:33.784439491Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:33.784481192Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, dataset_name: DatasetName { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dfb1f626-0f55-4385-87de-de59052d98d2, dataset_name: DatasetName { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:33.803168816Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:33.803212307Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: dfb1f626-0f55-4385-87de-de59052d98d2, dataset_name: DatasetName { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: adff3617-5320-4e42-8b10-1430a1eb0afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:33.824614445Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:33.82465849Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: adff3617-5320-4e42-8b10-1430a1eb0afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, dataset_name: DatasetName { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:33.845248411Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:33.845290104Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, dataset_name: DatasetName { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:33.865905056Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/6d2aa15e-618e-402e-b368-cf27fd8def68/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:33.865949486Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/75c788e1-45ef-4fcf-958d-db07331bef52/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:33.866184016Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:33.86629846Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:33.963504858Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:33.963547609Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:33.96355284Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:33.963556224Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:33.96356019Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:33.963564203Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:33.963568057Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:33.963572669Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:33.963576597Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:33.963580366Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:33.963584191Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:33.963605201Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:33.963610067Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:33.963613123Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:33.963616048Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:33.96361977Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:33.963622658Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:33.963625564Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:33.963629361Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:33.963710606Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"cb26b278-0bca-494c-8b86-9c68cc618ee9","remote_addr":"[fd00:1122:3344:101::1]:47495","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3\", TimeNotSynchronized), (\"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc\", TimeNotSynchronized), (\"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095\", TimeNotSynchronized), (\"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876\", TimeNotSynchronized), (\"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862\", TimeNotSynchronized), (\"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046\", TimeNotSynchronized), (\"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465\", TimeNotSynchronized), (\"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa\", TimeNotSynchronized), (\"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e\", TimeNotSynchronized), (\"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7\", TimeNotSynchronized), (\"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2\", TimeNotSynchronized), (\"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c\", TimeNotSynchronized), (\"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480\", TimeNotSynchronized), (\"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79\", TimeNotSynchronized), (\"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b\", TimeNotSynchronized), (\"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a\", TimeNotSynchronized), (\"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d\", TimeNotSynchronized), (\"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c\", TimeNotSynchronized), (\"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9\", TimeNotSynchronized)]","latency_us":514091,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:33.963996591Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:472","retry_after":"771.103491ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"cb26b278-0bca-494c-8b86-9c68cc618ee9\", \"content-length\": \"124\", \"date\": \"Thu, 25 Jul 2024 05:45:33 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"cb26b278-0bca-494c-8b86-9c68cc618ee9\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3f311cd5-ee38-4707-ab8a-9331491f8e7c, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6389ba5c-9d50-4fe4-b1a2-8be3c6861cb0, kind: Service { id: 3f311cd5-ee38-4707-ab8a-9331491f8e7c }, name: Name(\"ntp-3f311cd5-ee38-4707-ab8a-9331491f8e7c\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 235])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4f6a29ee-ca23-4fa8-be69-497060121028, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } } } }, OmicronZoneConfig { id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } } } }, OmicronZoneConfig { id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External } } } }, OmicronZoneConfig { id: b04db43b-dc24-43ce-9d97-32193f88e046, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f04fae05-ec27-43ac-ad4d-ab63c091bf1b, kind: Service { id: b04db43b-dc24-43ce-9d97-32193f88e046 }, name: Name(\"external-dns-b04db43b-dc24-43ce-9d97-32193f88e046\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 154])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 2c3f8453-233f-428f-93cc-0bfefb84929d, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: a6846ef5-8686-4c7e-ae28-d178a4029a85, kind: Service { id: 2c3f8453-233f-428f-93cc-0bfefb84929d }, name: Name(\"external-dns-2c3f8453-233f-428f-93cc-0bfefb84929d\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 164, 190])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: e4efbc31-e513-4290-82f5-c51d8afea0c7, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 735e2401-2ba5-4079-b4a0-7f0184716575, kind: Service { id: e4efbc31-e513-4290-82f5-c51d8afea0c7 }, name: Name(\"nexus-e4efbc31-e513-4290-82f5-c51d8afea0c7\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 154])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 85472347-dfd9-425e-bd98-9b81ebdf3b2b, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: f33554f0-0c2d-4b66-98d0-73365d475c4b, kind: Service { id: 85472347-dfd9-425e-bd98-9b81ebdf3b2b }, name: Name(\"nexus-85472347-dfd9-425e-bd98-9b81ebdf3b2b\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 244])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 221c5e7c-0832-4890-8b0a-f9ba08725876, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 9e8859b7-3d6e-4066-b58c-24fc535c1122, kind: Service { id: 221c5e7c-0832-4890-8b0a-f9ba08725876 }, name: Name(\"nexus-221c5e7c-0832-4890-8b0a-f9ba08725876\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 155])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4b4998f4-10de-49f2-9326-b482137c5fd3, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: a635cc62-7516-4fd9-b490-62d6de386095, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: ac9fa851-e7e1-4295-ac79-d6076526f465, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 1cb96976-2047-44b8-a79c-9fcd74d17bfa, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: a0b9fba0-c330-402f-9bc0-a7510786c98a, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } } } }, OmicronZoneConfig { id: d94d9602-d610-44ce-b9fc-06ded41a1480, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } } } }, OmicronZoneConfig { id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External } } } }, OmicronZoneConfig { id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External } } } }, OmicronZoneConfig { id: dfb1f626-0f55-4385-87de-de59052d98d2, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External } } } }, OmicronZoneConfig { id: adff3617-5320-4e42-8b10-1430a1eb0afc, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:34.738592032Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:434"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:34.738927835Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:34.738940941Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:34.762173247Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:34.762212633Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:34.782493674Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:34.782541825Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4f6a29ee-ca23-4fa8-be69-497060121028, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:34.802951893Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:34.802988635Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4f6a29ee-ca23-4fa8-be69-497060121028, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:34.822600406Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:34.822639924Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:34.843708179Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:34.843741705Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:34.863097159Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:34.863134285Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:34.881825398Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:34.881857959Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b04db43b-dc24-43ce-9d97-32193f88e046, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:34.901911405Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:34.901950059Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: b04db43b-dc24-43ce-9d97-32193f88e046, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2c3f8453-233f-428f-93cc-0bfefb84929d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:34.921542164Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:34.921580734Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2c3f8453-233f-428f-93cc-0bfefb84929d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:34.941439991Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:34.941476189Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a0b9fba0-c330-402f-9bc0-a7510786c98a, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:34.963086805Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:34.963128384Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: a0b9fba0-c330-402f-9bc0-a7510786c98a, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:34.982280654Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:34.982314694Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d94d9602-d610-44ce-b9fc-06ded41a1480, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:35.002328814Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:35.002364355Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d94d9602-d610-44ce-b9fc-06ded41a1480, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:35.02448871Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:35.024534283Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:35.052085185Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:35.052122742Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, dataset_name: DatasetName { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:35.072164372Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:35.072214549Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, dataset_name: DatasetName { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dfb1f626-0f55-4385-87de-de59052d98d2, dataset_name: DatasetName { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:35.09180849Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:35.091842654Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: dfb1f626-0f55-4385-87de-de59052d98d2, dataset_name: DatasetName { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: adff3617-5320-4e42-8b10-1430a1eb0afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:35.110326492Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:35.110359999Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: adff3617-5320-4e42-8b10-1430a1eb0afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, dataset_name: DatasetName { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:35.129076533Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:35.129117447Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, dataset_name: DatasetName { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:35.148474931Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/6d2aa15e-618e-402e-b368-cf27fd8def68/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:35.148512357Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/75c788e1-45ef-4fcf-958d-db07331bef52/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:35.148674286Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:35.148781604Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:35.248666351Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:35.248717895Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:35.24872547Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:35.248729417Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:35.248734451Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:35.248739514Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:35.248744709Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:35.248749799Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:35.248755106Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:35.248760159Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:35.248765259Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:35.248771981Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:35.2487771Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:35.248782406Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:35.248787328Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:35.248809575Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:35.248814928Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:35.248820104Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:35.248824886Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:35.248918291Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"f2a59ba6-96e6-4452-bf86-c93c2c12425d","remote_addr":"[fd00:1122:3344:101::1]:47495","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa\", TimeNotSynchronized), (\"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480\", TimeNotSynchronized), (\"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465\", TimeNotSynchronized), (\"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e\", TimeNotSynchronized), (\"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9\", TimeNotSynchronized), (\"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862\", TimeNotSynchronized), (\"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d\", TimeNotSynchronized), (\"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046\", TimeNotSynchronized), (\"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3\", TimeNotSynchronized), (\"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c\", TimeNotSynchronized), (\"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a\", TimeNotSynchronized), (\"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7\", TimeNotSynchronized), (\"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc\", TimeNotSynchronized), (\"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095\", TimeNotSynchronized), (\"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b\", TimeNotSynchronized), (\"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876\", TimeNotSynchronized), (\"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c\", TimeNotSynchronized), (\"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79\", TimeNotSynchronized), (\"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2\", TimeNotSynchronized)]","latency_us":508799,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:35.249154293Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:472","retry_after":"1.308676473s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f2a59ba6-96e6-4452-bf86-c93c2c12425d\", \"content-length\": \"124\", \"date\": \"Thu, 25 Jul 2024 05:45:34 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f2a59ba6-96e6-4452-bf86-c93c2c12425d\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3f311cd5-ee38-4707-ab8a-9331491f8e7c, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6389ba5c-9d50-4fe4-b1a2-8be3c6861cb0, kind: Service { id: 3f311cd5-ee38-4707-ab8a-9331491f8e7c }, name: Name(\"ntp-3f311cd5-ee38-4707-ab8a-9331491f8e7c\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 235])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4f6a29ee-ca23-4fa8-be69-497060121028, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } } } }, OmicronZoneConfig { id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } } } }, OmicronZoneConfig { id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External } } } }, OmicronZoneConfig { id: b04db43b-dc24-43ce-9d97-32193f88e046, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f04fae05-ec27-43ac-ad4d-ab63c091bf1b, kind: Service { id: b04db43b-dc24-43ce-9d97-32193f88e046 }, name: Name(\"external-dns-b04db43b-dc24-43ce-9d97-32193f88e046\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 154])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 2c3f8453-233f-428f-93cc-0bfefb84929d, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: a6846ef5-8686-4c7e-ae28-d178a4029a85, kind: Service { id: 2c3f8453-233f-428f-93cc-0bfefb84929d }, name: Name(\"external-dns-2c3f8453-233f-428f-93cc-0bfefb84929d\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 164, 190])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: e4efbc31-e513-4290-82f5-c51d8afea0c7, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 735e2401-2ba5-4079-b4a0-7f0184716575, kind: Service { id: e4efbc31-e513-4290-82f5-c51d8afea0c7 }, name: Name(\"nexus-e4efbc31-e513-4290-82f5-c51d8afea0c7\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 154])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 85472347-dfd9-425e-bd98-9b81ebdf3b2b, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: f33554f0-0c2d-4b66-98d0-73365d475c4b, kind: Service { id: 85472347-dfd9-425e-bd98-9b81ebdf3b2b }, name: Name(\"nexus-85472347-dfd9-425e-bd98-9b81ebdf3b2b\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 244])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 221c5e7c-0832-4890-8b0a-f9ba08725876, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 9e8859b7-3d6e-4066-b58c-24fc535c1122, kind: Service { id: 221c5e7c-0832-4890-8b0a-f9ba08725876 }, name: Name(\"nexus-221c5e7c-0832-4890-8b0a-f9ba08725876\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 155])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4b4998f4-10de-49f2-9326-b482137c5fd3, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: a635cc62-7516-4fd9-b490-62d6de386095, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: ac9fa851-e7e1-4295-ac79-d6076526f465, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 1cb96976-2047-44b8-a79c-9fcd74d17bfa, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: a0b9fba0-c330-402f-9bc0-a7510786c98a, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } } } }, OmicronZoneConfig { id: d94d9602-d610-44ce-b9fc-06ded41a1480, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } } } }, OmicronZoneConfig { id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External } } } }, OmicronZoneConfig { id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External } } } }, OmicronZoneConfig { id: dfb1f626-0f55-4385-87de-de59052d98d2, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External } } } }, OmicronZoneConfig { id: adff3617-5320-4e42-8b10-1430a1eb0afc, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:36.562659218Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:434"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:36.563040352Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:36.563049718Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:36.586426218Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:36.586472081Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:36.60798689Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:36.608101072Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4f6a29ee-ca23-4fa8-be69-497060121028, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:36.628238486Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:36.628289027Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4f6a29ee-ca23-4fa8-be69-497060121028, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:36.648636699Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:36.6486805Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:36.668434654Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:36.668470936Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:36.68770231Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:36.687754633Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:36.707354407Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:36.707393849Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b04db43b-dc24-43ce-9d97-32193f88e046, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:36.728467985Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:36.728525184Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: b04db43b-dc24-43ce-9d97-32193f88e046, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2c3f8453-233f-428f-93cc-0bfefb84929d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:36.749305269Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:36.749353645Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2c3f8453-233f-428f-93cc-0bfefb84929d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:36.770267703Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:36.770310454Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a0b9fba0-c330-402f-9bc0-a7510786c98a, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:36.791737341Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:36.791779305Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: a0b9fba0-c330-402f-9bc0-a7510786c98a, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:36.813043513Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:36.813088458Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d94d9602-d610-44ce-b9fc-06ded41a1480, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:36.833858306Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:36.833904132Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d94d9602-d610-44ce-b9fc-06ded41a1480, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:36.850214201Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","component":"MetricsManager","sled_id":"8fa4bc78-1026-41f0-b09d-c67224071376","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"1.946912249s"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:36.855955189Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:36.855966534Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:36.877311943Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:36.877357994Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, dataset_name: DatasetName { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:36.899571182Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:36.89961188Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, dataset_name: DatasetName { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dfb1f626-0f55-4385-87de-de59052d98d2, dataset_name: DatasetName { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:36.920274029Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:36.920318543Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: dfb1f626-0f55-4385-87de-de59052d98d2, dataset_name: DatasetName { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: adff3617-5320-4e42-8b10-1430a1eb0afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:36.940743457Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:36.940788206Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: adff3617-5320-4e42-8b10-1430a1eb0afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, dataset_name: DatasetName { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:36.96208538Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:36.962132041Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, dataset_name: DatasetName { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:36.983189885Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/6d2aa15e-618e-402e-b368-cf27fd8def68/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:36.983243427Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/75c788e1-45ef-4fcf-958d-db07331bef52/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:36.983499839Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:36.98363158Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:37.085624747Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:37.085672261Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:37.085686464Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:37.085690758Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:37.085694648Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:37.085698615Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:37.085701605Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:37.085705318Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:37.085708252Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:37.085711112Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:37.085713943Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:37.085716802Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:37.085720449Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:37.085724096Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:37.085726946Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:37.085730743Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:37.085734503Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:37.085749109Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:37.085752363Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:37.08578856Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"aa88a932-5faf-4460-b201-1e736dd2fd18","remote_addr":"[fd00:1122:3344:101::1]:47495","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7\", TimeNotSynchronized), (\"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b\", TimeNotSynchronized), (\"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095\", TimeNotSynchronized), (\"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465\", TimeNotSynchronized), (\"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d\", TimeNotSynchronized), (\"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a\", TimeNotSynchronized), (\"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c\", TimeNotSynchronized), (\"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa\", TimeNotSynchronized), (\"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480\", TimeNotSynchronized), (\"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79\", TimeNotSynchronized), (\"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046\", TimeNotSynchronized), (\"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e\", TimeNotSynchronized), (\"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c\", TimeNotSynchronized), (\"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2\", TimeNotSynchronized), (\"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc\", TimeNotSynchronized), (\"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9\", TimeNotSynchronized), (\"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3\", TimeNotSynchronized), (\"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876\", TimeNotSynchronized), (\"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862\", TimeNotSynchronized)]","latency_us":521566,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:37.085995837Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:472","retry_after":"1.325593067s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"aa88a932-5faf-4460-b201-1e736dd2fd18\", \"content-length\": \"124\", \"date\": \"Thu, 25 Jul 2024 05:45:36 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"aa88a932-5faf-4460-b201-1e736dd2fd18\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3f311cd5-ee38-4707-ab8a-9331491f8e7c, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6389ba5c-9d50-4fe4-b1a2-8be3c6861cb0, kind: Service { id: 3f311cd5-ee38-4707-ab8a-9331491f8e7c }, name: Name(\"ntp-3f311cd5-ee38-4707-ab8a-9331491f8e7c\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 235])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4f6a29ee-ca23-4fa8-be69-497060121028, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } } } }, OmicronZoneConfig { id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } } } }, OmicronZoneConfig { id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External } } } }, OmicronZoneConfig { id: b04db43b-dc24-43ce-9d97-32193f88e046, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f04fae05-ec27-43ac-ad4d-ab63c091bf1b, kind: Service { id: b04db43b-dc24-43ce-9d97-32193f88e046 }, name: Name(\"external-dns-b04db43b-dc24-43ce-9d97-32193f88e046\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 154])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 2c3f8453-233f-428f-93cc-0bfefb84929d, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: a6846ef5-8686-4c7e-ae28-d178a4029a85, kind: Service { id: 2c3f8453-233f-428f-93cc-0bfefb84929d }, name: Name(\"external-dns-2c3f8453-233f-428f-93cc-0bfefb84929d\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 164, 190])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: e4efbc31-e513-4290-82f5-c51d8afea0c7, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 735e2401-2ba5-4079-b4a0-7f0184716575, kind: Service { id: e4efbc31-e513-4290-82f5-c51d8afea0c7 }, name: Name(\"nexus-e4efbc31-e513-4290-82f5-c51d8afea0c7\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 154])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 85472347-dfd9-425e-bd98-9b81ebdf3b2b, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: f33554f0-0c2d-4b66-98d0-73365d475c4b, kind: Service { id: 85472347-dfd9-425e-bd98-9b81ebdf3b2b }, name: Name(\"nexus-85472347-dfd9-425e-bd98-9b81ebdf3b2b\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 244])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 221c5e7c-0832-4890-8b0a-f9ba08725876, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 9e8859b7-3d6e-4066-b58c-24fc535c1122, kind: Service { id: 221c5e7c-0832-4890-8b0a-f9ba08725876 }, name: Name(\"nexus-221c5e7c-0832-4890-8b0a-f9ba08725876\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 155])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4b4998f4-10de-49f2-9326-b482137c5fd3, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: a635cc62-7516-4fd9-b490-62d6de386095, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: ac9fa851-e7e1-4295-ac79-d6076526f465, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 1cb96976-2047-44b8-a79c-9fcd74d17bfa, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: a0b9fba0-c330-402f-9bc0-a7510786c98a, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } } } }, OmicronZoneConfig { id: d94d9602-d610-44ce-b9fc-06ded41a1480, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } } } }, OmicronZoneConfig { id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External } } } }, OmicronZoneConfig { id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External } } } }, OmicronZoneConfig { id: dfb1f626-0f55-4385-87de-de59052d98d2, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External } } } }, OmicronZoneConfig { id: adff3617-5320-4e42-8b10-1430a1eb0afc, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:38.416485798Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:434"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:38.417511795Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:38.417522212Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:38.439815362Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:38.439851897Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:38.460932489Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:38.460965369Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4f6a29ee-ca23-4fa8-be69-497060121028, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:38.47981681Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:38.479858736Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4f6a29ee-ca23-4fa8-be69-497060121028, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:38.501416408Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:38.501448528Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:38.523040221Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:38.52308571Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:38.545896111Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:38.545929581Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:38.567576379Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:38.567608836Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b04db43b-dc24-43ce-9d97-32193f88e046, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:38.590722395Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:38.59075669Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: b04db43b-dc24-43ce-9d97-32193f88e046, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2c3f8453-233f-428f-93cc-0bfefb84929d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:38.611676781Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:38.611712539Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2c3f8453-233f-428f-93cc-0bfefb84929d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:38.632107334Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:38.632139782Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a0b9fba0-c330-402f-9bc0-a7510786c98a, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:38.65314451Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:38.653176835Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: a0b9fba0-c330-402f-9bc0-a7510786c98a, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:38.676245574Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:38.676282062Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d94d9602-d610-44ce-b9fc-06ded41a1480, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:38.699585073Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:38.699622667Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d94d9602-d610-44ce-b9fc-06ded41a1480, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:38.721242446Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:38.721276403Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:38.743642353Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:38.743675861Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, dataset_name: DatasetName { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:38.764511621Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:38.764546179Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, dataset_name: DatasetName { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dfb1f626-0f55-4385-87de-de59052d98d2, dataset_name: DatasetName { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:38.785703991Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:38.78573808Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: dfb1f626-0f55-4385-87de-de59052d98d2, dataset_name: DatasetName { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: adff3617-5320-4e42-8b10-1430a1eb0afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:38.806122822Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:38.806156385Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: adff3617-5320-4e42-8b10-1430a1eb0afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, dataset_name: DatasetName { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:38.828777626Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:38.828816082Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, dataset_name: DatasetName { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:38.850861943Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/6d2aa15e-618e-402e-b368-cf27fd8def68/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:38.850901319Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/75c788e1-45ef-4fcf-958d-db07331bef52/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:38.85113822Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:38.851238375Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:38.952662493Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:38.952730108Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:38.95273727Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:38.952744621Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:38.952750668Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:38.952768602Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:38.952775146Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:38.952781325Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:38.952789125Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:38.952796709Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:38.952802775Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:38.952810163Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:38.952817822Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:38.95282401Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:38.952831492Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:38.952837295Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:38.952843492Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:38.952850898Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:38.952858304Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:38.952949704Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"8eb183c4-cd58-4a8a-81d4-67679171915a","remote_addr":"[fd00:1122:3344:101::1]:47495","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3\", TimeNotSynchronized), (\"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465\", TimeNotSynchronized), (\"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa\", TimeNotSynchronized), (\"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc\", TimeNotSynchronized), (\"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9\", TimeNotSynchronized), (\"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480\", TimeNotSynchronized), (\"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79\", TimeNotSynchronized), (\"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046\", TimeNotSynchronized), (\"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b\", TimeNotSynchronized), (\"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a\", TimeNotSynchronized), (\"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c\", TimeNotSynchronized), (\"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876\", TimeNotSynchronized), (\"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c\", TimeNotSynchronized), (\"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d\", TimeNotSynchronized), (\"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e\", TimeNotSynchronized), (\"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2\", TimeNotSynchronized), (\"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7\", TimeNotSynchronized), (\"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862\", TimeNotSynchronized), (\"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095\", TimeNotSynchronized)]","latency_us":534864,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:38.953211385Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:472","retry_after":"2.307591668s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"8eb183c4-cd58-4a8a-81d4-67679171915a\", \"content-length\": \"124\", \"date\": \"Thu, 25 Jul 2024 05:45:38 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"8eb183c4-cd58-4a8a-81d4-67679171915a\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3f311cd5-ee38-4707-ab8a-9331491f8e7c, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6389ba5c-9d50-4fe4-b1a2-8be3c6861cb0, kind: Service { id: 3f311cd5-ee38-4707-ab8a-9331491f8e7c }, name: Name(\"ntp-3f311cd5-ee38-4707-ab8a-9331491f8e7c\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 235])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4f6a29ee-ca23-4fa8-be69-497060121028, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } } } }, OmicronZoneConfig { id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } } } }, OmicronZoneConfig { id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External } } } }, OmicronZoneConfig { id: b04db43b-dc24-43ce-9d97-32193f88e046, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f04fae05-ec27-43ac-ad4d-ab63c091bf1b, kind: Service { id: b04db43b-dc24-43ce-9d97-32193f88e046 }, name: Name(\"external-dns-b04db43b-dc24-43ce-9d97-32193f88e046\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 154])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 2c3f8453-233f-428f-93cc-0bfefb84929d, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: a6846ef5-8686-4c7e-ae28-d178a4029a85, kind: Service { id: 2c3f8453-233f-428f-93cc-0bfefb84929d }, name: Name(\"external-dns-2c3f8453-233f-428f-93cc-0bfefb84929d\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 164, 190])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: e4efbc31-e513-4290-82f5-c51d8afea0c7, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 735e2401-2ba5-4079-b4a0-7f0184716575, kind: Service { id: e4efbc31-e513-4290-82f5-c51d8afea0c7 }, name: Name(\"nexus-e4efbc31-e513-4290-82f5-c51d8afea0c7\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 154])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 85472347-dfd9-425e-bd98-9b81ebdf3b2b, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: f33554f0-0c2d-4b66-98d0-73365d475c4b, kind: Service { id: 85472347-dfd9-425e-bd98-9b81ebdf3b2b }, name: Name(\"nexus-85472347-dfd9-425e-bd98-9b81ebdf3b2b\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 244])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 221c5e7c-0832-4890-8b0a-f9ba08725876, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 9e8859b7-3d6e-4066-b58c-24fc535c1122, kind: Service { id: 221c5e7c-0832-4890-8b0a-f9ba08725876 }, name: Name(\"nexus-221c5e7c-0832-4890-8b0a-f9ba08725876\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 155])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4b4998f4-10de-49f2-9326-b482137c5fd3, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: a635cc62-7516-4fd9-b490-62d6de386095, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: ac9fa851-e7e1-4295-ac79-d6076526f465, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 1cb96976-2047-44b8-a79c-9fcd74d17bfa, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: a0b9fba0-c330-402f-9bc0-a7510786c98a, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } } } }, OmicronZoneConfig { id: d94d9602-d610-44ce-b9fc-06ded41a1480, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } } } }, OmicronZoneConfig { id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External } } } }, OmicronZoneConfig { id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External } } } }, OmicronZoneConfig { id: dfb1f626-0f55-4385-87de-de59052d98d2, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External } } } }, OmicronZoneConfig { id: adff3617-5320-4e42-8b10-1430a1eb0afc, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:41.268069305Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:434"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:41.268996118Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:41.269002868Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:41.291979866Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:41.292021651Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:41.312795988Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:41.312835898Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4f6a29ee-ca23-4fa8-be69-497060121028, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:41.333098329Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:41.333139383Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4f6a29ee-ca23-4fa8-be69-497060121028, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:41.351995972Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:41.352028401Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:41.371701098Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:41.371743868Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:41.391355317Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:41.391396765Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:41.409119527Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:41.409159784Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b04db43b-dc24-43ce-9d97-32193f88e046, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:41.430478224Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:41.430515406Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: b04db43b-dc24-43ce-9d97-32193f88e046, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2c3f8453-233f-428f-93cc-0bfefb84929d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:41.449809157Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:41.449846143Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2c3f8453-233f-428f-93cc-0bfefb84929d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:41.469330179Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:41.469367886Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a0b9fba0-c330-402f-9bc0-a7510786c98a, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:41.48973591Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:41.489773336Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: a0b9fba0-c330-402f-9bc0-a7510786c98a, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:41.508891351Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:41.508928392Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d94d9602-d610-44ce-b9fc-06ded41a1480, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:41.528549405Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:41.528589446Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d94d9602-d610-44ce-b9fc-06ded41a1480, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:41.547975356Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:41.548020939Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:41.569894521Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:41.569937066Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, dataset_name: DatasetName { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:41.589366057Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:41.589405545Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, dataset_name: DatasetName { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dfb1f626-0f55-4385-87de-de59052d98d2, dataset_name: DatasetName { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:41.608610612Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:41.608649069Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: dfb1f626-0f55-4385-87de-de59052d98d2, dataset_name: DatasetName { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: adff3617-5320-4e42-8b10-1430a1eb0afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:41.627946378Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:41.627984085Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: adff3617-5320-4e42-8b10-1430a1eb0afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, dataset_name: DatasetName { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:41.648436314Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:41.648496409Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, dataset_name: DatasetName { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:41.667373968Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/6d2aa15e-618e-402e-b368-cf27fd8def68/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:41.667418154Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/75c788e1-45ef-4fcf-958d-db07331bef52/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:41.667622018Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:41.667797015Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:41.766564917Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:41.766617747Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:41.766625819Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:41.766632494Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:41.766637838Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:41.766643304Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:41.766648854Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:41.766653655Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:41.766659992Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:41.766672976Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:41.766678517Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:41.766685426Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:41.766690751Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:41.766695974Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:41.76670113Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:41.76670759Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:41.766712887Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:41.766716459Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:41.766721821Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:41.766795689Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"075b6451-20b8-46d9-8400-ac134b045446","remote_addr":"[fd00:1122:3344:101::1]:47495","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a\", TimeNotSynchronized), (\"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e\", TimeNotSynchronized), (\"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9\", TimeNotSynchronized), (\"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876\", TimeNotSynchronized), (\"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7\", TimeNotSynchronized), (\"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3\", TimeNotSynchronized), (\"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465\", TimeNotSynchronized), (\"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c\", TimeNotSynchronized), (\"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa\", TimeNotSynchronized), (\"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480\", TimeNotSynchronized), (\"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79\", TimeNotSynchronized), (\"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095\", TimeNotSynchronized), (\"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c\", TimeNotSynchronized), (\"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2\", TimeNotSynchronized), (\"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc\", TimeNotSynchronized), (\"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046\", TimeNotSynchronized), (\"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d\", TimeNotSynchronized), (\"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b\", TimeNotSynchronized), (\"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862\", TimeNotSynchronized)]","latency_us":497265,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:41.767028055Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:472","retry_after":"2.208510459s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"075b6451-20b8-46d9-8400-ac134b045446\", \"content-length\": \"124\", \"date\": \"Thu, 25 Jul 2024 05:45:41 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"075b6451-20b8-46d9-8400-ac134b045446\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3f311cd5-ee38-4707-ab8a-9331491f8e7c, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6389ba5c-9d50-4fe4-b1a2-8be3c6861cb0, kind: Service { id: 3f311cd5-ee38-4707-ab8a-9331491f8e7c }, name: Name(\"ntp-3f311cd5-ee38-4707-ab8a-9331491f8e7c\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 235])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4f6a29ee-ca23-4fa8-be69-497060121028, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } } } }, OmicronZoneConfig { id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } } } }, OmicronZoneConfig { id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External } } } }, OmicronZoneConfig { id: b04db43b-dc24-43ce-9d97-32193f88e046, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f04fae05-ec27-43ac-ad4d-ab63c091bf1b, kind: Service { id: b04db43b-dc24-43ce-9d97-32193f88e046 }, name: Name(\"external-dns-b04db43b-dc24-43ce-9d97-32193f88e046\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 154])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 2c3f8453-233f-428f-93cc-0bfefb84929d, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: a6846ef5-8686-4c7e-ae28-d178a4029a85, kind: Service { id: 2c3f8453-233f-428f-93cc-0bfefb84929d }, name: Name(\"external-dns-2c3f8453-233f-428f-93cc-0bfefb84929d\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 164, 190])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: e4efbc31-e513-4290-82f5-c51d8afea0c7, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 735e2401-2ba5-4079-b4a0-7f0184716575, kind: Service { id: e4efbc31-e513-4290-82f5-c51d8afea0c7 }, name: Name(\"nexus-e4efbc31-e513-4290-82f5-c51d8afea0c7\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 154])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 85472347-dfd9-425e-bd98-9b81ebdf3b2b, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: f33554f0-0c2d-4b66-98d0-73365d475c4b, kind: Service { id: 85472347-dfd9-425e-bd98-9b81ebdf3b2b }, name: Name(\"nexus-85472347-dfd9-425e-bd98-9b81ebdf3b2b\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 244])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 221c5e7c-0832-4890-8b0a-f9ba08725876, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 9e8859b7-3d6e-4066-b58c-24fc535c1122, kind: Service { id: 221c5e7c-0832-4890-8b0a-f9ba08725876 }, name: Name(\"nexus-221c5e7c-0832-4890-8b0a-f9ba08725876\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 155])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4b4998f4-10de-49f2-9326-b482137c5fd3, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: a635cc62-7516-4fd9-b490-62d6de386095, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: ac9fa851-e7e1-4295-ac79-d6076526f465, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 1cb96976-2047-44b8-a79c-9fcd74d17bfa, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: a0b9fba0-c330-402f-9bc0-a7510786c98a, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } } } }, OmicronZoneConfig { id: d94d9602-d610-44ce-b9fc-06ded41a1480, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } } } }, OmicronZoneConfig { id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External } } } }, OmicronZoneConfig { id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External } } } }, OmicronZoneConfig { id: dfb1f626-0f55-4385-87de-de59052d98d2, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External } } } }, OmicronZoneConfig { id: adff3617-5320-4e42-8b10-1430a1eb0afc, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:43.982140656Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:434"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:43.983048618Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:43.983061958Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:44.005848264Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:44.005888231Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:44.028531881Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:44.028573732Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4f6a29ee-ca23-4fa8-be69-497060121028, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:44.051361524Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:44.051406198Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4f6a29ee-ca23-4fa8-be69-497060121028, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:44.071387091Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:44.071427685Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:44.093544005Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:44.093585931Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:44.11436968Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:44.114425557Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:44.133873155Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:44.133912277Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b04db43b-dc24-43ce-9d97-32193f88e046, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:44.154895529Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:44.154931858Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: b04db43b-dc24-43ce-9d97-32193f88e046, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2c3f8453-233f-428f-93cc-0bfefb84929d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:44.175750041Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:44.17579176Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2c3f8453-233f-428f-93cc-0bfefb84929d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:44.195312661Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:44.195345427Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a0b9fba0-c330-402f-9bc0-a7510786c98a, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:44.215365853Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:44.21541077Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: a0b9fba0-c330-402f-9bc0-a7510786c98a, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:44.236219849Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:44.236263134Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d94d9602-d610-44ce-b9fc-06ded41a1480, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:44.256201278Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:44.256238329Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d94d9602-d610-44ce-b9fc-06ded41a1480, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:44.277606097Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:44.277640654Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:44.297151512Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:44.297194441Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, dataset_name: DatasetName { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:44.315515233Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:44.315555049Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, dataset_name: DatasetName { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dfb1f626-0f55-4385-87de-de59052d98d2, dataset_name: DatasetName { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:44.333984708Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:44.334023437Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: dfb1f626-0f55-4385-87de-de59052d98d2, dataset_name: DatasetName { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: adff3617-5320-4e42-8b10-1430a1eb0afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:44.354332449Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:44.354375013Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: adff3617-5320-4e42-8b10-1430a1eb0afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, dataset_name: DatasetName { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:44.374850373Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:44.374896734Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, dataset_name: DatasetName { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:44.393325051Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/6d2aa15e-618e-402e-b368-cf27fd8def68/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:44.393385156Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/75c788e1-45ef-4fcf-958d-db07331bef52/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:44.393576598Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:44.393703023Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:44.491711107Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:44.491765745Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:44.491773536Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:44.491778871Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:44.491783961Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:44.491789099Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:44.49179404Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:44.491800227Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:44.491805149Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:44.491810118Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:44.491816399Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:44.49182151Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:44.491825203Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:44.491837888Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:44.491842903Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:44.491846541Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:44.491852738Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:44.491856Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:44.4918608Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:44.491912543Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"2da27270-e47f-4dcb-9ec1-aa04995abffe","remote_addr":"[fd00:1122:3344:101::1]:47495","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9\", TimeNotSynchronized), (\"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c\", TimeNotSynchronized), (\"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465\", TimeNotSynchronized), (\"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b\", TimeNotSynchronized), (\"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc\", TimeNotSynchronized), (\"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2\", TimeNotSynchronized), (\"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7\", TimeNotSynchronized), (\"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862\", TimeNotSynchronized), (\"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a\", TimeNotSynchronized), (\"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095\", TimeNotSynchronized), (\"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3\", TimeNotSynchronized), (\"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79\", TimeNotSynchronized), (\"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c\", TimeNotSynchronized), (\"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046\", TimeNotSynchronized), (\"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d\", TimeNotSynchronized), (\"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876\", TimeNotSynchronized), (\"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa\", TimeNotSynchronized), (\"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480\", TimeNotSynchronized), (\"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e\", TimeNotSynchronized)]","latency_us":508275,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:44.492088168Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:472","retry_after":"3.3310263s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"2da27270-e47f-4dcb-9ec1-aa04995abffe\", \"content-length\": \"124\", \"date\": \"Thu, 25 Jul 2024 05:45:44 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"2da27270-e47f-4dcb-9ec1-aa04995abffe\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3f311cd5-ee38-4707-ab8a-9331491f8e7c, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6389ba5c-9d50-4fe4-b1a2-8be3c6861cb0, kind: Service { id: 3f311cd5-ee38-4707-ab8a-9331491f8e7c }, name: Name(\"ntp-3f311cd5-ee38-4707-ab8a-9331491f8e7c\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 235])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4f6a29ee-ca23-4fa8-be69-497060121028, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } } } }, OmicronZoneConfig { id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } } } }, OmicronZoneConfig { id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External } } } }, OmicronZoneConfig { id: b04db43b-dc24-43ce-9d97-32193f88e046, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f04fae05-ec27-43ac-ad4d-ab63c091bf1b, kind: Service { id: b04db43b-dc24-43ce-9d97-32193f88e046 }, name: Name(\"external-dns-b04db43b-dc24-43ce-9d97-32193f88e046\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 154])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 2c3f8453-233f-428f-93cc-0bfefb84929d, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: a6846ef5-8686-4c7e-ae28-d178a4029a85, kind: Service { id: 2c3f8453-233f-428f-93cc-0bfefb84929d }, name: Name(\"external-dns-2c3f8453-233f-428f-93cc-0bfefb84929d\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 164, 190])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: e4efbc31-e513-4290-82f5-c51d8afea0c7, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 735e2401-2ba5-4079-b4a0-7f0184716575, kind: Service { id: e4efbc31-e513-4290-82f5-c51d8afea0c7 }, name: Name(\"nexus-e4efbc31-e513-4290-82f5-c51d8afea0c7\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 154])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 85472347-dfd9-425e-bd98-9b81ebdf3b2b, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: f33554f0-0c2d-4b66-98d0-73365d475c4b, kind: Service { id: 85472347-dfd9-425e-bd98-9b81ebdf3b2b }, name: Name(\"nexus-85472347-dfd9-425e-bd98-9b81ebdf3b2b\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 244])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 221c5e7c-0832-4890-8b0a-f9ba08725876, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 9e8859b7-3d6e-4066-b58c-24fc535c1122, kind: Service { id: 221c5e7c-0832-4890-8b0a-f9ba08725876 }, name: Name(\"nexus-221c5e7c-0832-4890-8b0a-f9ba08725876\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 155])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4b4998f4-10de-49f2-9326-b482137c5fd3, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: a635cc62-7516-4fd9-b490-62d6de386095, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: ac9fa851-e7e1-4295-ac79-d6076526f465, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 1cb96976-2047-44b8-a79c-9fcd74d17bfa, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: a0b9fba0-c330-402f-9bc0-a7510786c98a, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } } } }, OmicronZoneConfig { id: d94d9602-d610-44ce-b9fc-06ded41a1480, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } } } }, OmicronZoneConfig { id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External } } } }, OmicronZoneConfig { id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External } } } }, OmicronZoneConfig { id: dfb1f626-0f55-4385-87de-de59052d98d2, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External } } } }, OmicronZoneConfig { id: adff3617-5320-4e42-8b10-1430a1eb0afc, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:47.833471263Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:434"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:47.83393664Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:47.833945603Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:47.857161689Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:47.857205078Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:47.876884876Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:47.876928968Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4f6a29ee-ca23-4fa8-be69-497060121028, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:47.895426801Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:47.895471361Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4f6a29ee-ca23-4fa8-be69-497060121028, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:47.915237159Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:47.915285273Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:47.936136765Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:47.936178392Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:47.95720881Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:47.957250203Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:47.97810165Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:47.978148582Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b04db43b-dc24-43ce-9d97-32193f88e046, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:47.99805163Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:47.998099903Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: b04db43b-dc24-43ce-9d97-32193f88e046, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2c3f8453-233f-428f-93cc-0bfefb84929d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:48.01850673Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:48.018545281Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2c3f8453-233f-428f-93cc-0bfefb84929d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:48.037391425Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:48.037427304Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a0b9fba0-c330-402f-9bc0-a7510786c98a, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:48.057315106Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:48.057354566Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: a0b9fba0-c330-402f-9bc0-a7510786c98a, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:48.075943389Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:48.0760016Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d94d9602-d610-44ce-b9fc-06ded41a1480, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:48.097555537Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:48.09759405Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d94d9602-d610-44ce-b9fc-06ded41a1480, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:48.117303417Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:48.117342746Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:48.13764934Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:48.137690891Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, dataset_name: DatasetName { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:48.156597894Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:48.156639304Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, dataset_name: DatasetName { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dfb1f626-0f55-4385-87de-de59052d98d2, dataset_name: DatasetName { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:48.176826317Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:48.176869143Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: dfb1f626-0f55-4385-87de-de59052d98d2, dataset_name: DatasetName { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: adff3617-5320-4e42-8b10-1430a1eb0afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:48.195699056Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:48.195747591Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: adff3617-5320-4e42-8b10-1430a1eb0afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, dataset_name: DatasetName { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:48.215826406Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:48.215887533Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, dataset_name: DatasetName { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:48.236298798Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/6d2aa15e-618e-402e-b368-cf27fd8def68/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:48.236355481Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/75c788e1-45ef-4fcf-958d-db07331bef52/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:48.236644393Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:48.236777661Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:48.337375077Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:48.337450988Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:48.337476592Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:48.337489408Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:48.337496036Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:48.337507353Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:48.337513512Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:48.337518687Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:48.337522643Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:48.337527115Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:48.337532703Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:48.337537784Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:48.337541685Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:48.337548135Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:48.337553432Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:48.337557351Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:48.337567297Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:48.337582719Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:48.337586564Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:48.337604958Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"c62904cb-95a8-4c83-8234-e6aead661d22","remote_addr":"[fd00:1122:3344:101::1]:47495","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b\", TimeNotSynchronized), (\"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3\", TimeNotSynchronized), (\"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095\", TimeNotSynchronized), (\"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465\", TimeNotSynchronized), (\"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa\", TimeNotSynchronized), (\"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a\", TimeNotSynchronized), (\"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c\", TimeNotSynchronized), (\"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480\", TimeNotSynchronized), (\"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d\", TimeNotSynchronized), (\"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046\", TimeNotSynchronized), (\"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7\", TimeNotSynchronized), (\"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876\", TimeNotSynchronized), (\"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862\", TimeNotSynchronized), (\"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c\", TimeNotSynchronized), (\"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79\", TimeNotSynchronized), (\"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e\", TimeNotSynchronized), (\"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2\", TimeNotSynchronized), (\"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc\", TimeNotSynchronized), (\"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9\", TimeNotSynchronized)]","latency_us":502552,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:48.33855171Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:472","retry_after":"4.301329722s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c62904cb-95a8-4c83-8234-e6aead661d22\", \"content-length\": \"124\", \"date\": \"Thu, 25 Jul 2024 05:45:48 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"c62904cb-95a8-4c83-8234-e6aead661d22\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:52.04551802Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:44104"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:52.045577047Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"aa92d8bc-e2fe-4d99-9383-5c41d3532dd7","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:44104","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":29,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3f311cd5-ee38-4707-ab8a-9331491f8e7c, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6389ba5c-9d50-4fe4-b1a2-8be3c6861cb0, kind: Service { id: 3f311cd5-ee38-4707-ab8a-9331491f8e7c }, name: Name(\"ntp-3f311cd5-ee38-4707-ab8a-9331491f8e7c\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 235])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4f6a29ee-ca23-4fa8-be69-497060121028, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } } } }, OmicronZoneConfig { id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } } } }, OmicronZoneConfig { id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External } } } }, OmicronZoneConfig { id: b04db43b-dc24-43ce-9d97-32193f88e046, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f04fae05-ec27-43ac-ad4d-ab63c091bf1b, kind: Service { id: b04db43b-dc24-43ce-9d97-32193f88e046 }, name: Name(\"external-dns-b04db43b-dc24-43ce-9d97-32193f88e046\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 154])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 2c3f8453-233f-428f-93cc-0bfefb84929d, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: a6846ef5-8686-4c7e-ae28-d178a4029a85, kind: Service { id: 2c3f8453-233f-428f-93cc-0bfefb84929d }, name: Name(\"external-dns-2c3f8453-233f-428f-93cc-0bfefb84929d\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 164, 190])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: e4efbc31-e513-4290-82f5-c51d8afea0c7, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 735e2401-2ba5-4079-b4a0-7f0184716575, kind: Service { id: e4efbc31-e513-4290-82f5-c51d8afea0c7 }, name: Name(\"nexus-e4efbc31-e513-4290-82f5-c51d8afea0c7\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 154])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 85472347-dfd9-425e-bd98-9b81ebdf3b2b, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: f33554f0-0c2d-4b66-98d0-73365d475c4b, kind: Service { id: 85472347-dfd9-425e-bd98-9b81ebdf3b2b }, name: Name(\"nexus-85472347-dfd9-425e-bd98-9b81ebdf3b2b\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 244])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 221c5e7c-0832-4890-8b0a-f9ba08725876, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 9e8859b7-3d6e-4066-b58c-24fc535c1122, kind: Service { id: 221c5e7c-0832-4890-8b0a-f9ba08725876 }, name: Name(\"nexus-221c5e7c-0832-4890-8b0a-f9ba08725876\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 155])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4b4998f4-10de-49f2-9326-b482137c5fd3, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: a635cc62-7516-4fd9-b490-62d6de386095, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: ac9fa851-e7e1-4295-ac79-d6076526f465, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 1cb96976-2047-44b8-a79c-9fcd74d17bfa, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: a0b9fba0-c330-402f-9bc0-a7510786c98a, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } } } }, OmicronZoneConfig { id: d94d9602-d610-44ce-b9fc-06ded41a1480, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } } } }, OmicronZoneConfig { id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External } } } }, OmicronZoneConfig { id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External } } } }, OmicronZoneConfig { id: dfb1f626-0f55-4385-87de-de59052d98d2, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External } } } }, OmicronZoneConfig { id: adff3617-5320-4e42-8b10-1430a1eb0afc, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:52.653149909Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:434"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:52.653503758Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:52.653510995Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:52.675661092Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:52.678310785Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:52.695777009Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:52.695837554Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4f6a29ee-ca23-4fa8-be69-497060121028, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:52.718076724Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:52.720561297Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4f6a29ee-ca23-4fa8-be69-497060121028, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:52.740284417Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:52.740336927Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:52.760424427Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:52.760482272Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:52.780135365Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:52.780199379Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:52.800411077Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:52.800461573Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b04db43b-dc24-43ce-9d97-32193f88e046, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:52.819455627Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:52.819510744Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: b04db43b-dc24-43ce-9d97-32193f88e046, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2c3f8453-233f-428f-93cc-0bfefb84929d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:52.839611118Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:52.839660441Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2c3f8453-233f-428f-93cc-0bfefb84929d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:52.86027895Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:52.860330898Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a0b9fba0-c330-402f-9bc0-a7510786c98a, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:52.88053688Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:52.880594424Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: a0b9fba0-c330-402f-9bc0-a7510786c98a, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:52.902067595Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:52.902192505Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d94d9602-d610-44ce-b9fc-06ded41a1480, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:52.922987609Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:52.923039519Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d94d9602-d610-44ce-b9fc-06ded41a1480, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:52.942975512Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:52.94303841Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:52.963067353Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:52.96311721Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, dataset_name: DatasetName { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:52.984163851Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:52.984215471Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, dataset_name: DatasetName { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dfb1f626-0f55-4385-87de-de59052d98d2, dataset_name: DatasetName { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:53.005703263Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:53.005770118Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: dfb1f626-0f55-4385-87de-de59052d98d2, dataset_name: DatasetName { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: adff3617-5320-4e42-8b10-1430a1eb0afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:53.026273929Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:53.026323618Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: adff3617-5320-4e42-8b10-1430a1eb0afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, dataset_name: DatasetName { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:53.048135903Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:53.048184063Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, dataset_name: DatasetName { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:53.068856233Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/6d2aa15e-618e-402e-b368-cf27fd8def68/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:53.068915148Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/75c788e1-45ef-4fcf-958d-db07331bef52/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:53.06912249Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:53.069247565Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:53.16793257Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:53.167996678Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:53.168027072Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:53.168044407Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:53.168051372Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:53.168057064Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:53.168060955Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:53.168071155Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:53.168075776Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:53.168082808Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:53.168087992Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:53.168093065Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:53.168097115Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:53.168101015Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:53.168106096Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:53.168115227Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:53.168120187Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:53.168124377Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:53.168131868Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:53.168137456Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"05221f7a-196d-4d62-8f39-24c8bd9d6670","remote_addr":"[fd00:1122:3344:101::1]:47495","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa\", TimeNotSynchronized), (\"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9\", TimeNotSynchronized), (\"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095\", TimeNotSynchronized), (\"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79\", TimeNotSynchronized), (\"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d\", TimeNotSynchronized), (\"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046\", TimeNotSynchronized), (\"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7\", TimeNotSynchronized), (\"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465\", TimeNotSynchronized), (\"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e\", TimeNotSynchronized), (\"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c\", TimeNotSynchronized), (\"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c\", TimeNotSynchronized), (\"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc\", TimeNotSynchronized), (\"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b\", TimeNotSynchronized), (\"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876\", TimeNotSynchronized), (\"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a\", TimeNotSynchronized), (\"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480\", TimeNotSynchronized), (\"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3\", TimeNotSynchronized), (\"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2\", TimeNotSynchronized), (\"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862\", TimeNotSynchronized)]","latency_us":513447,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:53.168356414Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:472","retry_after":"4.205581186s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"05221f7a-196d-4d62-8f39-24c8bd9d6670\", \"content-length\": \"124\", \"date\": \"Thu, 25 Jul 2024 05:45:53 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"05221f7a-196d-4d62-8f39-24c8bd9d6670\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:53.899919861Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","component":"MetricsManager","sled_id":"8fa4bc78-1026-41f0-b09d-c67224071376","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"2.014013351s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3f311cd5-ee38-4707-ab8a-9331491f8e7c, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6389ba5c-9d50-4fe4-b1a2-8be3c6861cb0, kind: Service { id: 3f311cd5-ee38-4707-ab8a-9331491f8e7c }, name: Name(\"ntp-3f311cd5-ee38-4707-ab8a-9331491f8e7c\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 235])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4f6a29ee-ca23-4fa8-be69-497060121028, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } } } }, OmicronZoneConfig { id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } } } }, OmicronZoneConfig { id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External } } } }, OmicronZoneConfig { id: b04db43b-dc24-43ce-9d97-32193f88e046, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f04fae05-ec27-43ac-ad4d-ab63c091bf1b, kind: Service { id: b04db43b-dc24-43ce-9d97-32193f88e046 }, name: Name(\"external-dns-b04db43b-dc24-43ce-9d97-32193f88e046\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 154])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 2c3f8453-233f-428f-93cc-0bfefb84929d, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: a6846ef5-8686-4c7e-ae28-d178a4029a85, kind: Service { id: 2c3f8453-233f-428f-93cc-0bfefb84929d }, name: Name(\"external-dns-2c3f8453-233f-428f-93cc-0bfefb84929d\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 164, 190])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: e4efbc31-e513-4290-82f5-c51d8afea0c7, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 735e2401-2ba5-4079-b4a0-7f0184716575, kind: Service { id: e4efbc31-e513-4290-82f5-c51d8afea0c7 }, name: Name(\"nexus-e4efbc31-e513-4290-82f5-c51d8afea0c7\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 154])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 85472347-dfd9-425e-bd98-9b81ebdf3b2b, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: f33554f0-0c2d-4b66-98d0-73365d475c4b, kind: Service { id: 85472347-dfd9-425e-bd98-9b81ebdf3b2b }, name: Name(\"nexus-85472347-dfd9-425e-bd98-9b81ebdf3b2b\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 244])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 221c5e7c-0832-4890-8b0a-f9ba08725876, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 9e8859b7-3d6e-4066-b58c-24fc535c1122, kind: Service { id: 221c5e7c-0832-4890-8b0a-f9ba08725876 }, name: Name(\"nexus-221c5e7c-0832-4890-8b0a-f9ba08725876\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 155])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4b4998f4-10de-49f2-9326-b482137c5fd3, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: a635cc62-7516-4fd9-b490-62d6de386095, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: ac9fa851-e7e1-4295-ac79-d6076526f465, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 1cb96976-2047-44b8-a79c-9fcd74d17bfa, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: a0b9fba0-c330-402f-9bc0-a7510786c98a, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } } } }, OmicronZoneConfig { id: d94d9602-d610-44ce-b9fc-06ded41a1480, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } } } }, OmicronZoneConfig { id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External } } } }, OmicronZoneConfig { id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External } } } }, OmicronZoneConfig { id: dfb1f626-0f55-4385-87de-de59052d98d2, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External } } } }, OmicronZoneConfig { id: adff3617-5320-4e42-8b10-1430a1eb0afc, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:57.386316203Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:434"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:57.387041987Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:57.387050218Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:57.411370457Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:57.411443424Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:57.43392892Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:57.436241864Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4f6a29ee-ca23-4fa8-be69-497060121028, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:57.454239155Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:57.454287906Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4f6a29ee-ca23-4fa8-be69-497060121028, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:57.472847712Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:57.472925057Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:57.491800552Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:57.491851628Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:57.511069084Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:57.513387601Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:57.531769833Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:57.531829788Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b04db43b-dc24-43ce-9d97-32193f88e046, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:57.55184426Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:57.554211258Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: b04db43b-dc24-43ce-9d97-32193f88e046, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2c3f8453-233f-428f-93cc-0bfefb84929d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:57.572569878Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:57.572620214Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2c3f8453-233f-428f-93cc-0bfefb84929d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:57.592607665Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:57.59266581Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a0b9fba0-c330-402f-9bc0-a7510786c98a, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:57.612505621Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:57.612555628Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: a0b9fba0-c330-402f-9bc0-a7510786c98a, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:57.634201115Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:57.634254095Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d94d9602-d610-44ce-b9fc-06ded41a1480, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:57.655592011Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:57.655651113Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d94d9602-d610-44ce-b9fc-06ded41a1480, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:57.676646189Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:57.676704578Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:57.696881059Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:57.696930533Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, dataset_name: DatasetName { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:57.716670508Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:57.716737813Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, dataset_name: DatasetName { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dfb1f626-0f55-4385-87de-de59052d98d2, dataset_name: DatasetName { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:57.737332346Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:57.737384669Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: dfb1f626-0f55-4385-87de-de59052d98d2, dataset_name: DatasetName { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: adff3617-5320-4e42-8b10-1430a1eb0afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:57.756486319Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:57.756543818Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: adff3617-5320-4e42-8b10-1430a1eb0afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, dataset_name: DatasetName { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:57.777991836Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:57.778047141Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, dataset_name: DatasetName { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:57.800522864Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/6d2aa15e-618e-402e-b368-cf27fd8def68/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:57.800590694Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/75c788e1-45ef-4fcf-958d-db07331bef52/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:57.800767172Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:57.800899917Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:57.89754598Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:57.897722941Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:57.897743941Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:57.897755971Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:57.897787077Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:57.897799078Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:57.897805734Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:57.897814396Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:57.897822534Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:57.897838491Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:57.897845157Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:57.897857082Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:57.897881064Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:57.89788728Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:57.897900039Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:57.897906359Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:57.897915715Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:57.897924143Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:57.897932524Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:45:57.89794716Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"bb4662a5-720b-48d4-a51c-a90573724811","remote_addr":"[fd00:1122:3344:101::1]:47495","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095\", TimeNotSynchronized), (\"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc\", TimeNotSynchronized), (\"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79\", TimeNotSynchronized), (\"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7\", TimeNotSynchronized), (\"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b\", TimeNotSynchronized), (\"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa\", TimeNotSynchronized), (\"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c\", TimeNotSynchronized), (\"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480\", TimeNotSynchronized), (\"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465\", TimeNotSynchronized), (\"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862\", TimeNotSynchronized), (\"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e\", TimeNotSynchronized), (\"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2\", TimeNotSynchronized), (\"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9\", TimeNotSynchronized), (\"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d\", TimeNotSynchronized), (\"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876\", TimeNotSynchronized), (\"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046\", TimeNotSynchronized), (\"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3\", TimeNotSynchronized), (\"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a\", TimeNotSynchronized), (\"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c\", TimeNotSynchronized)]","latency_us":510046,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:45:57.898093075Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:472","retry_after":"5.815637744s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"bb4662a5-720b-48d4-a51c-a90573724811\", \"content-length\": \"124\", \"date\": \"Thu, 25 Jul 2024 05:45:57 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"bb4662a5-720b-48d4-a51c-a90573724811\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3f311cd5-ee38-4707-ab8a-9331491f8e7c, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6389ba5c-9d50-4fe4-b1a2-8be3c6861cb0, kind: Service { id: 3f311cd5-ee38-4707-ab8a-9331491f8e7c }, name: Name(\"ntp-3f311cd5-ee38-4707-ab8a-9331491f8e7c\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 235])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4f6a29ee-ca23-4fa8-be69-497060121028, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } } } }, OmicronZoneConfig { id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } } } }, OmicronZoneConfig { id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External } } } }, OmicronZoneConfig { id: b04db43b-dc24-43ce-9d97-32193f88e046, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f04fae05-ec27-43ac-ad4d-ab63c091bf1b, kind: Service { id: b04db43b-dc24-43ce-9d97-32193f88e046 }, name: Name(\"external-dns-b04db43b-dc24-43ce-9d97-32193f88e046\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 154])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 2c3f8453-233f-428f-93cc-0bfefb84929d, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: a6846ef5-8686-4c7e-ae28-d178a4029a85, kind: Service { id: 2c3f8453-233f-428f-93cc-0bfefb84929d }, name: Name(\"external-dns-2c3f8453-233f-428f-93cc-0bfefb84929d\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 164, 190])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: e4efbc31-e513-4290-82f5-c51d8afea0c7, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 735e2401-2ba5-4079-b4a0-7f0184716575, kind: Service { id: e4efbc31-e513-4290-82f5-c51d8afea0c7 }, name: Name(\"nexus-e4efbc31-e513-4290-82f5-c51d8afea0c7\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 154])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 85472347-dfd9-425e-bd98-9b81ebdf3b2b, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: f33554f0-0c2d-4b66-98d0-73365d475c4b, kind: Service { id: 85472347-dfd9-425e-bd98-9b81ebdf3b2b }, name: Name(\"nexus-85472347-dfd9-425e-bd98-9b81ebdf3b2b\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 244])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 221c5e7c-0832-4890-8b0a-f9ba08725876, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 9e8859b7-3d6e-4066-b58c-24fc535c1122, kind: Service { id: 221c5e7c-0832-4890-8b0a-f9ba08725876 }, name: Name(\"nexus-221c5e7c-0832-4890-8b0a-f9ba08725876\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 155])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4b4998f4-10de-49f2-9326-b482137c5fd3, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: a635cc62-7516-4fd9-b490-62d6de386095, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: ac9fa851-e7e1-4295-ac79-d6076526f465, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 1cb96976-2047-44b8-a79c-9fcd74d17bfa, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: a0b9fba0-c330-402f-9bc0-a7510786c98a, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } } } }, OmicronZoneConfig { id: d94d9602-d610-44ce-b9fc-06ded41a1480, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } } } }, OmicronZoneConfig { id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External } } } }, OmicronZoneConfig { id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External } } } }, OmicronZoneConfig { id: dfb1f626-0f55-4385-87de-de59052d98d2, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External } } } }, OmicronZoneConfig { id: adff3617-5320-4e42-8b10-1430a1eb0afc, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:03.730614076Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:434"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:03.731601105Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:03.731618985Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:03.755745025Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:03.755799955Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:03.77773713Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:03.777803957Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4f6a29ee-ca23-4fa8-be69-497060121028, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:03.798884071Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:03.798937388Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4f6a29ee-ca23-4fa8-be69-497060121028, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:03.818855941Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:03.818919646Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:03.838078466Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:03.83813152Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:03.858967529Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:03.859018568Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:03.879548875Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:03.879593763Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b04db43b-dc24-43ce-9d97-32193f88e046, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:03.898955646Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:03.899004032Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: b04db43b-dc24-43ce-9d97-32193f88e046, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2c3f8453-233f-428f-93cc-0bfefb84929d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:03.9230396Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:03.923101908Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2c3f8453-233f-428f-93cc-0bfefb84929d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:03.944692028Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:03.944743451Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a0b9fba0-c330-402f-9bc0-a7510786c98a, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:03.966358325Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:03.966415318Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: a0b9fba0-c330-402f-9bc0-a7510786c98a, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:03.987573221Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:03.987629603Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d94d9602-d610-44ce-b9fc-06ded41a1480, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:04.008642167Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:04.008690009Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d94d9602-d610-44ce-b9fc-06ded41a1480, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:04.029742037Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:04.029807269Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:04.050445796Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:04.050497023Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, dataset_name: DatasetName { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:04.071191787Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:04.071242657Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, dataset_name: DatasetName { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dfb1f626-0f55-4385-87de-de59052d98d2, dataset_name: DatasetName { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:04.09194117Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:04.09199429Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: dfb1f626-0f55-4385-87de-de59052d98d2, dataset_name: DatasetName { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: adff3617-5320-4e42-8b10-1430a1eb0afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:04.112483502Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:04.112539566Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: adff3617-5320-4e42-8b10-1430a1eb0afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, dataset_name: DatasetName { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:04.133119197Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:04.13316581Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, dataset_name: DatasetName { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:04.153687485Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/6d2aa15e-618e-402e-b368-cf27fd8def68/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:04.153771422Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/75c788e1-45ef-4fcf-958d-db07331bef52/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:04.153977405Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:04.154102367Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:04.256695904Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:04.256771534Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:04.256852699Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:04.256864991Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:04.256871534Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:04.256876822Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:04.256882353Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:04.256889066Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:04.256918954Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:04.256925301Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:04.256932388Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:04.256939486Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:04.256944755Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:04.256957543Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:04.256962014Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:04.256967733Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:04.256972158Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:04.256976912Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:04.256982275Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:04.257046626Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"f5e81b75-df7e-418c-abc1-94882621c903","remote_addr":"[fd00:1122:3344:101::1]:47495","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7\", TimeNotSynchronized), (\"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876\", TimeNotSynchronized), (\"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465\", TimeNotSynchronized), (\"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2\", TimeNotSynchronized), (\"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046\", TimeNotSynchronized), (\"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3\", TimeNotSynchronized), (\"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c\", TimeNotSynchronized), (\"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a\", TimeNotSynchronized), (\"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa\", TimeNotSynchronized), (\"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9\", TimeNotSynchronized), (\"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095\", TimeNotSynchronized), (\"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc\", TimeNotSynchronized), (\"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d\", TimeNotSynchronized), (\"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862\", TimeNotSynchronized), (\"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e\", TimeNotSynchronized), (\"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b\", TimeNotSynchronized), (\"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480\", TimeNotSynchronized), (\"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79\", TimeNotSynchronized), (\"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c\", TimeNotSynchronized)]","latency_us":524813,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:04.257238902Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:472","retry_after":"4.059541631s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f5e81b75-df7e-418c-abc1-94882621c903\", \"content-length\": \"124\", \"date\": \"Thu, 25 Jul 2024 05:46:04 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f5e81b75-df7e-418c-abc1-94882621c903\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3f311cd5-ee38-4707-ab8a-9331491f8e7c, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6389ba5c-9d50-4fe4-b1a2-8be3c6861cb0, kind: Service { id: 3f311cd5-ee38-4707-ab8a-9331491f8e7c }, name: Name(\"ntp-3f311cd5-ee38-4707-ab8a-9331491f8e7c\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 235])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4f6a29ee-ca23-4fa8-be69-497060121028, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } } } }, OmicronZoneConfig { id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } } } }, OmicronZoneConfig { id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External } } } }, OmicronZoneConfig { id: b04db43b-dc24-43ce-9d97-32193f88e046, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f04fae05-ec27-43ac-ad4d-ab63c091bf1b, kind: Service { id: b04db43b-dc24-43ce-9d97-32193f88e046 }, name: Name(\"external-dns-b04db43b-dc24-43ce-9d97-32193f88e046\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 154])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 2c3f8453-233f-428f-93cc-0bfefb84929d, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: a6846ef5-8686-4c7e-ae28-d178a4029a85, kind: Service { id: 2c3f8453-233f-428f-93cc-0bfefb84929d }, name: Name(\"external-dns-2c3f8453-233f-428f-93cc-0bfefb84929d\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 164, 190])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: e4efbc31-e513-4290-82f5-c51d8afea0c7, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 735e2401-2ba5-4079-b4a0-7f0184716575, kind: Service { id: e4efbc31-e513-4290-82f5-c51d8afea0c7 }, name: Name(\"nexus-e4efbc31-e513-4290-82f5-c51d8afea0c7\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 154])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 85472347-dfd9-425e-bd98-9b81ebdf3b2b, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: f33554f0-0c2d-4b66-98d0-73365d475c4b, kind: Service { id: 85472347-dfd9-425e-bd98-9b81ebdf3b2b }, name: Name(\"nexus-85472347-dfd9-425e-bd98-9b81ebdf3b2b\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 244])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 221c5e7c-0832-4890-8b0a-f9ba08725876, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 9e8859b7-3d6e-4066-b58c-24fc535c1122, kind: Service { id: 221c5e7c-0832-4890-8b0a-f9ba08725876 }, name: Name(\"nexus-221c5e7c-0832-4890-8b0a-f9ba08725876\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 155])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4b4998f4-10de-49f2-9326-b482137c5fd3, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: a635cc62-7516-4fd9-b490-62d6de386095, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: ac9fa851-e7e1-4295-ac79-d6076526f465, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 1cb96976-2047-44b8-a79c-9fcd74d17bfa, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: a0b9fba0-c330-402f-9bc0-a7510786c98a, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } } } }, OmicronZoneConfig { id: d94d9602-d610-44ce-b9fc-06ded41a1480, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } } } }, OmicronZoneConfig { id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External } } } }, OmicronZoneConfig { id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External } } } }, OmicronZoneConfig { id: dfb1f626-0f55-4385-87de-de59052d98d2, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External } } } }, OmicronZoneConfig { id: adff3617-5320-4e42-8b10-1430a1eb0afc, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:08.32959659Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:434"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:08.329964247Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:08.329977241Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:08.354618145Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:08.354679355Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:08.376121993Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:08.376171195Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4f6a29ee-ca23-4fa8-be69-497060121028, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:08.398759335Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:08.398810036Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4f6a29ee-ca23-4fa8-be69-497060121028, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:08.419422796Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:08.41947901Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:08.441951438Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:08.4420233Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:08.462716384Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:08.462773807Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:08.483672951Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:08.48373222Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b04db43b-dc24-43ce-9d97-32193f88e046, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:08.504821053Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:08.504881269Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: b04db43b-dc24-43ce-9d97-32193f88e046, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2c3f8453-233f-428f-93cc-0bfefb84929d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:08.52540336Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:08.525459433Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2c3f8453-233f-428f-93cc-0bfefb84929d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:08.545993791Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:08.546052564Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a0b9fba0-c330-402f-9bc0-a7510786c98a, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:08.565836927Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:08.565884787Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: a0b9fba0-c330-402f-9bc0-a7510786c98a, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:08.586477621Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:08.586596398Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d94d9602-d610-44ce-b9fc-06ded41a1480, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:08.608885041Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:08.608948633Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d94d9602-d610-44ce-b9fc-06ded41a1480, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:08.632374755Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:08.63245345Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:08.656356663Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:08.656428674Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, dataset_name: DatasetName { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:08.679595665Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:08.679649207Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, dataset_name: DatasetName { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dfb1f626-0f55-4385-87de-de59052d98d2, dataset_name: DatasetName { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:08.700884983Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:08.700938375Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: dfb1f626-0f55-4385-87de-de59052d98d2, dataset_name: DatasetName { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: adff3617-5320-4e42-8b10-1430a1eb0afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:08.722151867Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:08.722203947Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: adff3617-5320-4e42-8b10-1430a1eb0afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, dataset_name: DatasetName { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:08.744495424Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:08.744543837Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, dataset_name: DatasetName { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:08.765367608Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/6d2aa15e-618e-402e-b368-cf27fd8def68/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:08.765425969Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/75c788e1-45ef-4fcf-958d-db07331bef52/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:08.765629861Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:08.765751336Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:08.867364435Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:08.867447509Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:08.867454203Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:08.867459715Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:08.86746534Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:08.86747091Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:08.867477876Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:08.867484588Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:08.867490344Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:08.867496982Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:08.867502747Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:08.867510483Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:08.86753784Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:08.867545631Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:08.867557368Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:08.867561315Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:08.867566734Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:08.867570737Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:08.867576193Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:08.867645082Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"dd86c9dd-7184-4622-9a76-979ea913ab6e","remote_addr":"[fd00:1122:3344:101::1]:47495","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c\", TimeNotSynchronized), (\"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046\", TimeNotSynchronized), (\"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2\", TimeNotSynchronized), (\"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862\", TimeNotSynchronized), (\"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc\", TimeNotSynchronized), (\"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9\", TimeNotSynchronized), (\"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79\", TimeNotSynchronized), (\"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3\", TimeNotSynchronized), (\"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c\", TimeNotSynchronized), (\"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d\", TimeNotSynchronized), (\"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b\", TimeNotSynchronized), (\"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465\", TimeNotSynchronized), (\"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa\", TimeNotSynchronized), (\"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7\", TimeNotSynchronized), (\"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876\", TimeNotSynchronized), (\"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095\", TimeNotSynchronized), (\"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a\", TimeNotSynchronized), (\"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480\", TimeNotSynchronized), (\"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e\", TimeNotSynchronized)]","latency_us":536449,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:08.867844559Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:472","retry_after":"8.248718398s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"dd86c9dd-7184-4622-9a76-979ea913ab6e\", \"content-length\": \"124\", \"date\": \"Thu, 25 Jul 2024 05:46:08 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"dd86c9dd-7184-4622-9a76-979ea913ab6e\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:10.995395278Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","component":"MetricsManager","sled_id":"8fa4bc78-1026-41f0-b09d-c67224071376","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"6.112851105s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3f311cd5-ee38-4707-ab8a-9331491f8e7c, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6389ba5c-9d50-4fe4-b1a2-8be3c6861cb0, kind: Service { id: 3f311cd5-ee38-4707-ab8a-9331491f8e7c }, name: Name(\"ntp-3f311cd5-ee38-4707-ab8a-9331491f8e7c\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 235])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4f6a29ee-ca23-4fa8-be69-497060121028, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } } } }, OmicronZoneConfig { id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } } } }, OmicronZoneConfig { id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External } } } }, OmicronZoneConfig { id: b04db43b-dc24-43ce-9d97-32193f88e046, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f04fae05-ec27-43ac-ad4d-ab63c091bf1b, kind: Service { id: b04db43b-dc24-43ce-9d97-32193f88e046 }, name: Name(\"external-dns-b04db43b-dc24-43ce-9d97-32193f88e046\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 154])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 2c3f8453-233f-428f-93cc-0bfefb84929d, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: a6846ef5-8686-4c7e-ae28-d178a4029a85, kind: Service { id: 2c3f8453-233f-428f-93cc-0bfefb84929d }, name: Name(\"external-dns-2c3f8453-233f-428f-93cc-0bfefb84929d\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 164, 190])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: e4efbc31-e513-4290-82f5-c51d8afea0c7, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 735e2401-2ba5-4079-b4a0-7f0184716575, kind: Service { id: e4efbc31-e513-4290-82f5-c51d8afea0c7 }, name: Name(\"nexus-e4efbc31-e513-4290-82f5-c51d8afea0c7\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 154])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 85472347-dfd9-425e-bd98-9b81ebdf3b2b, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: f33554f0-0c2d-4b66-98d0-73365d475c4b, kind: Service { id: 85472347-dfd9-425e-bd98-9b81ebdf3b2b }, name: Name(\"nexus-85472347-dfd9-425e-bd98-9b81ebdf3b2b\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 244])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 221c5e7c-0832-4890-8b0a-f9ba08725876, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 9e8859b7-3d6e-4066-b58c-24fc535c1122, kind: Service { id: 221c5e7c-0832-4890-8b0a-f9ba08725876 }, name: Name(\"nexus-221c5e7c-0832-4890-8b0a-f9ba08725876\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 155])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4b4998f4-10de-49f2-9326-b482137c5fd3, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: a635cc62-7516-4fd9-b490-62d6de386095, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: ac9fa851-e7e1-4295-ac79-d6076526f465, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 1cb96976-2047-44b8-a79c-9fcd74d17bfa, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: a0b9fba0-c330-402f-9bc0-a7510786c98a, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } } } }, OmicronZoneConfig { id: d94d9602-d610-44ce-b9fc-06ded41a1480, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } } } }, OmicronZoneConfig { id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External } } } }, OmicronZoneConfig { id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External } } } }, OmicronZoneConfig { id: dfb1f626-0f55-4385-87de-de59052d98d2, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External } } } }, OmicronZoneConfig { id: adff3617-5320-4e42-8b10-1430a1eb0afc, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:17.155356622Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:434"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:17.156281444Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:17.156288589Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:17.179677006Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:17.179734785Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:17.199536302Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:17.19960446Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4f6a29ee-ca23-4fa8-be69-497060121028, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:17.218676102Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:17.21873541Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4f6a29ee-ca23-4fa8-be69-497060121028, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:17.237758393Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:17.23780646Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:17.256171114Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:17.25623195Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:17.274745879Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:17.277143982Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:17.292889362Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:17.292945079Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b04db43b-dc24-43ce-9d97-32193f88e046, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:17.310967106Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:17.31101401Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: b04db43b-dc24-43ce-9d97-32193f88e046, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2c3f8453-233f-428f-93cc-0bfefb84929d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:17.329025282Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:17.329076471Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2c3f8453-233f-428f-93cc-0bfefb84929d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:17.347753882Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:17.34779653Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a0b9fba0-c330-402f-9bc0-a7510786c98a, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:17.366763203Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:17.366825379Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: a0b9fba0-c330-402f-9bc0-a7510786c98a, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:17.385719041Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:17.385774654Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d94d9602-d610-44ce-b9fc-06ded41a1480, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:17.403913911Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:17.403960299Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d94d9602-d610-44ce-b9fc-06ded41a1480, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:17.421923999Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:17.421972694Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:17.440706738Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:17.440751008Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, dataset_name: DatasetName { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:17.45891852Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:17.458963213Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, dataset_name: DatasetName { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dfb1f626-0f55-4385-87de-de59052d98d2, dataset_name: DatasetName { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:17.477510306Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:17.477628151Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: dfb1f626-0f55-4385-87de-de59052d98d2, dataset_name: DatasetName { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: adff3617-5320-4e42-8b10-1430a1eb0afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:17.496225475Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:17.496279177Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: adff3617-5320-4e42-8b10-1430a1eb0afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, dataset_name: DatasetName { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:17.51452399Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:17.514585754Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, dataset_name: DatasetName { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:17.532991008Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/6d2aa15e-618e-402e-b368-cf27fd8def68/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:17.533041935Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/75c788e1-45ef-4fcf-958d-db07331bef52/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:17.533241345Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:17.53336793Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:17.628572252Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:17.628644216Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:17.628665338Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:17.628675332Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:17.628687267Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:17.62869728Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:17.628706383Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:17.628721571Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:17.628729352Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:17.628735109Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:17.628744418Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:17.628753484Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:17.62876254Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:17.628787797Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:17.628795832Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:17.628801935Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:17.628809633Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:17.628814995Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:17.628820451Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:17.628826508Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"2e6446d6-f287-46f7-88cc-04f20b8283a5","remote_addr":"[fd00:1122:3344:101::1]:47495","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d\", TimeNotSynchronized), (\"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c\", TimeNotSynchronized), (\"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2\", TimeNotSynchronized), (\"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9\", TimeNotSynchronized), (\"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b\", TimeNotSynchronized), (\"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a\", TimeNotSynchronized), (\"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046\", TimeNotSynchronized), (\"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c\", TimeNotSynchronized), (\"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79\", TimeNotSynchronized), (\"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3\", TimeNotSynchronized), (\"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862\", TimeNotSynchronized), (\"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa\", TimeNotSynchronized), (\"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc\", TimeNotSynchronized), (\"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465\", TimeNotSynchronized), (\"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7\", TimeNotSynchronized), (\"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e\", TimeNotSynchronized), (\"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095\", TimeNotSynchronized), (\"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876\", TimeNotSynchronized), (\"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480\", TimeNotSynchronized)]","latency_us":472094,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:17.629111799Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:472","retry_after":"7.529878821s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"2e6446d6-f287-46f7-88cc-04f20b8283a5\", \"content-length\": \"124\", \"date\": \"Thu, 25 Jul 2024 05:46:17 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"2e6446d6-f287-46f7-88cc-04f20b8283a5\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:22.161990468Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:55774"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:22.162871503Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"8a9cc9b9-1a77-476a-87da-f4c79c77463a","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:55774","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":817,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3f311cd5-ee38-4707-ab8a-9331491f8e7c, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6389ba5c-9d50-4fe4-b1a2-8be3c6861cb0, kind: Service { id: 3f311cd5-ee38-4707-ab8a-9331491f8e7c }, name: Name(\"ntp-3f311cd5-ee38-4707-ab8a-9331491f8e7c\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 235])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4f6a29ee-ca23-4fa8-be69-497060121028, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } } } }, OmicronZoneConfig { id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } } } }, OmicronZoneConfig { id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External } } } }, OmicronZoneConfig { id: b04db43b-dc24-43ce-9d97-32193f88e046, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f04fae05-ec27-43ac-ad4d-ab63c091bf1b, kind: Service { id: b04db43b-dc24-43ce-9d97-32193f88e046 }, name: Name(\"external-dns-b04db43b-dc24-43ce-9d97-32193f88e046\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 154])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 2c3f8453-233f-428f-93cc-0bfefb84929d, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: a6846ef5-8686-4c7e-ae28-d178a4029a85, kind: Service { id: 2c3f8453-233f-428f-93cc-0bfefb84929d }, name: Name(\"external-dns-2c3f8453-233f-428f-93cc-0bfefb84929d\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 164, 190])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: e4efbc31-e513-4290-82f5-c51d8afea0c7, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 735e2401-2ba5-4079-b4a0-7f0184716575, kind: Service { id: e4efbc31-e513-4290-82f5-c51d8afea0c7 }, name: Name(\"nexus-e4efbc31-e513-4290-82f5-c51d8afea0c7\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 154])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 85472347-dfd9-425e-bd98-9b81ebdf3b2b, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: f33554f0-0c2d-4b66-98d0-73365d475c4b, kind: Service { id: 85472347-dfd9-425e-bd98-9b81ebdf3b2b }, name: Name(\"nexus-85472347-dfd9-425e-bd98-9b81ebdf3b2b\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 244])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 221c5e7c-0832-4890-8b0a-f9ba08725876, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 9e8859b7-3d6e-4066-b58c-24fc535c1122, kind: Service { id: 221c5e7c-0832-4890-8b0a-f9ba08725876 }, name: Name(\"nexus-221c5e7c-0832-4890-8b0a-f9ba08725876\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 155])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4b4998f4-10de-49f2-9326-b482137c5fd3, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: a635cc62-7516-4fd9-b490-62d6de386095, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: ac9fa851-e7e1-4295-ac79-d6076526f465, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 1cb96976-2047-44b8-a79c-9fcd74d17bfa, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: a0b9fba0-c330-402f-9bc0-a7510786c98a, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } } } }, OmicronZoneConfig { id: d94d9602-d610-44ce-b9fc-06ded41a1480, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } } } }, OmicronZoneConfig { id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External } } } }, OmicronZoneConfig { id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External } } } }, OmicronZoneConfig { id: dfb1f626-0f55-4385-87de-de59052d98d2, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External } } } }, OmicronZoneConfig { id: adff3617-5320-4e42-8b10-1430a1eb0afc, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:25.180763894Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:434"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:25.181317791Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:25.181327401Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:25.204141693Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:25.20420415Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:25.223260197Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:25.223318839Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4f6a29ee-ca23-4fa8-be69-497060121028, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:25.242332949Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:25.24238622Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4f6a29ee-ca23-4fa8-be69-497060121028, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:25.260181247Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:25.260226351Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:25.277976364Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:25.278036936Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:25.295413937Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:25.295471211Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:25.314128951Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:25.314178714Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b04db43b-dc24-43ce-9d97-32193f88e046, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:25.331512674Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:25.331553213Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: b04db43b-dc24-43ce-9d97-32193f88e046, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2c3f8453-233f-428f-93cc-0bfefb84929d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:25.351193441Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:25.351246795Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2c3f8453-233f-428f-93cc-0bfefb84929d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:25.371197954Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:25.371241202Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a0b9fba0-c330-402f-9bc0-a7510786c98a, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:25.391909389Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:25.391966353Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: a0b9fba0-c330-402f-9bc0-a7510786c98a, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:25.415260107Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:25.415323419Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d94d9602-d610-44ce-b9fc-06ded41a1480, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:25.435894573Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:25.435955034Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d94d9602-d610-44ce-b9fc-06ded41a1480, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:25.456624029Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:25.456672414Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:25.475119989Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:25.475171684Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, dataset_name: DatasetName { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:25.494791536Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:25.494846287Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, dataset_name: DatasetName { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dfb1f626-0f55-4385-87de-de59052d98d2, dataset_name: DatasetName { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:25.513993613Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:25.514045176Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: dfb1f626-0f55-4385-87de-de59052d98d2, dataset_name: DatasetName { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: adff3617-5320-4e42-8b10-1430a1eb0afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:25.532689925Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:25.532740852Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: adff3617-5320-4e42-8b10-1430a1eb0afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, dataset_name: DatasetName { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:25.551719027Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:25.551770375Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, dataset_name: DatasetName { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:25.570410084Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/6d2aa15e-618e-402e-b368-cf27fd8def68/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:25.570535974Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/75c788e1-45ef-4fcf-958d-db07331bef52/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:25.570732357Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:25.57086016Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:25.667482942Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:25.667572673Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:25.667585245Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:25.66760437Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:25.667613108Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:25.667620665Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:25.667634437Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:25.667642021Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:25.66765104Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:25.667660594Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:25.667669894Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:25.667679016Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:25.667686807Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:25.667695628Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:25.667791428Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:25.66779919Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:25.667804685Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:25.667812344Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:25.667821841Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:25.667895306Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"07eb67e9-b3c7-497d-b5da-b034ee2fe4a4","remote_addr":"[fd00:1122:3344:101::1]:47495","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c\", TimeNotSynchronized), (\"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480\", TimeNotSynchronized), (\"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79\", TimeNotSynchronized), (\"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095\", TimeNotSynchronized), (\"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc\", TimeNotSynchronized), (\"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862\", TimeNotSynchronized), (\"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e\", TimeNotSynchronized), (\"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9\", TimeNotSynchronized), (\"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3\", TimeNotSynchronized), (\"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876\", TimeNotSynchronized), (\"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b\", TimeNotSynchronized), (\"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa\", TimeNotSynchronized), (\"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a\", TimeNotSynchronized), (\"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2\", TimeNotSynchronized), (\"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7\", TimeNotSynchronized), (\"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c\", TimeNotSynchronized), (\"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046\", TimeNotSynchronized), (\"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d\", TimeNotSynchronized), (\"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465\", TimeNotSynchronized)]","latency_us":485558,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:25.668103295Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:472","retry_after":"6.546953853s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"07eb67e9-b3c7-497d-b5da-b034ee2fe4a4\", \"content-length\": \"124\", \"date\": \"Thu, 25 Jul 2024 05:46:25 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"07eb67e9-b3c7-497d-b5da-b034ee2fe4a4\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:32.198810022Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","component":"MetricsManager","sled_id":"8fa4bc78-1026-41f0-b09d-c67224071376","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"21.728864219s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3f311cd5-ee38-4707-ab8a-9331491f8e7c, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6389ba5c-9d50-4fe4-b1a2-8be3c6861cb0, kind: Service { id: 3f311cd5-ee38-4707-ab8a-9331491f8e7c }, name: Name(\"ntp-3f311cd5-ee38-4707-ab8a-9331491f8e7c\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 235])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4f6a29ee-ca23-4fa8-be69-497060121028, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } } } }, OmicronZoneConfig { id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } } } }, OmicronZoneConfig { id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External } } } }, OmicronZoneConfig { id: b04db43b-dc24-43ce-9d97-32193f88e046, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f04fae05-ec27-43ac-ad4d-ab63c091bf1b, kind: Service { id: b04db43b-dc24-43ce-9d97-32193f88e046 }, name: Name(\"external-dns-b04db43b-dc24-43ce-9d97-32193f88e046\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 154])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 2c3f8453-233f-428f-93cc-0bfefb84929d, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: a6846ef5-8686-4c7e-ae28-d178a4029a85, kind: Service { id: 2c3f8453-233f-428f-93cc-0bfefb84929d }, name: Name(\"external-dns-2c3f8453-233f-428f-93cc-0bfefb84929d\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 164, 190])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: e4efbc31-e513-4290-82f5-c51d8afea0c7, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 735e2401-2ba5-4079-b4a0-7f0184716575, kind: Service { id: e4efbc31-e513-4290-82f5-c51d8afea0c7 }, name: Name(\"nexus-e4efbc31-e513-4290-82f5-c51d8afea0c7\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 154])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 85472347-dfd9-425e-bd98-9b81ebdf3b2b, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: f33554f0-0c2d-4b66-98d0-73365d475c4b, kind: Service { id: 85472347-dfd9-425e-bd98-9b81ebdf3b2b }, name: Name(\"nexus-85472347-dfd9-425e-bd98-9b81ebdf3b2b\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 244])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 221c5e7c-0832-4890-8b0a-f9ba08725876, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 9e8859b7-3d6e-4066-b58c-24fc535c1122, kind: Service { id: 221c5e7c-0832-4890-8b0a-f9ba08725876 }, name: Name(\"nexus-221c5e7c-0832-4890-8b0a-f9ba08725876\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 155])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4b4998f4-10de-49f2-9326-b482137c5fd3, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: a635cc62-7516-4fd9-b490-62d6de386095, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: ac9fa851-e7e1-4295-ac79-d6076526f465, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 1cb96976-2047-44b8-a79c-9fcd74d17bfa, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: a0b9fba0-c330-402f-9bc0-a7510786c98a, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } } } }, OmicronZoneConfig { id: d94d9602-d610-44ce-b9fc-06ded41a1480, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } } } }, OmicronZoneConfig { id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External } } } }, OmicronZoneConfig { id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External } } } }, OmicronZoneConfig { id: dfb1f626-0f55-4385-87de-de59052d98d2, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External } } } }, OmicronZoneConfig { id: adff3617-5320-4e42-8b10-1430a1eb0afc, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:32.234100254Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:434"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:32.234621274Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:32.234633527Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:32.256876195Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:32.259107076Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:32.276577377Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:32.276710986Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4f6a29ee-ca23-4fa8-be69-497060121028, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:32.294614426Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:32.29473565Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4f6a29ee-ca23-4fa8-be69-497060121028, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:32.312571941Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:32.31261727Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:32.330010168Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:32.330066109Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:32.348435636Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:32.348489122Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:32.366372494Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:32.366441861Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b04db43b-dc24-43ce-9d97-32193f88e046, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:32.385329356Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:32.385382485Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: b04db43b-dc24-43ce-9d97-32193f88e046, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2c3f8453-233f-428f-93cc-0bfefb84929d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:32.403254539Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:32.403300121Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2c3f8453-233f-428f-93cc-0bfefb84929d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:32.42181635Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:32.424163817Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a0b9fba0-c330-402f-9bc0-a7510786c98a, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:32.440126771Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:32.440178522Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: a0b9fba0-c330-402f-9bc0-a7510786c98a, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:32.458437384Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:32.458492876Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d94d9602-d610-44ce-b9fc-06ded41a1480, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:32.478112491Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:32.47823689Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d94d9602-d610-44ce-b9fc-06ded41a1480, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:32.496505264Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:32.49656054Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:32.514862344Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:32.51491088Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, dataset_name: DatasetName { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:32.533881532Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:32.53393904Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, dataset_name: DatasetName { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dfb1f626-0f55-4385-87de-de59052d98d2, dataset_name: DatasetName { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:32.551703956Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:32.551755632Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: dfb1f626-0f55-4385-87de-de59052d98d2, dataset_name: DatasetName { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: adff3617-5320-4e42-8b10-1430a1eb0afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:32.57020587Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:32.570251705Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: adff3617-5320-4e42-8b10-1430a1eb0afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, dataset_name: DatasetName { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:32.589208257Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:32.589251842Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, dataset_name: DatasetName { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:32.607850299Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/6d2aa15e-618e-402e-b368-cf27fd8def68/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:32.607902125Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/75c788e1-45ef-4fcf-958d-db07331bef52/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:32.608141793Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:32.608277163Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:32.701706301Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:32.70184358Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:32.701852834Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:32.701862041Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:32.701870066Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:32.701879103Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:32.701888544Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:32.701897442Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:32.701908092Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:32.701916023Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:32.70192551Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:32.701934998Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:32.701944421Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:32.701953908Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:32.701965027Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:32.701970736Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:32.7019926Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:32.701999894Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:32.70200895Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:32.702070039Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"140661b4-14dc-49a6-a219-d42e1a73fe5e","remote_addr":"[fd00:1122:3344:101::1]:47495","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a\", TimeNotSynchronized), (\"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862\", TimeNotSynchronized), (\"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095\", TimeNotSynchronized), (\"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e\", TimeNotSynchronized), (\"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3\", TimeNotSynchronized), (\"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465\", TimeNotSynchronized), (\"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa\", TimeNotSynchronized), (\"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480\", TimeNotSynchronized), (\"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79\", TimeNotSynchronized), (\"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7\", TimeNotSynchronized), (\"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b\", TimeNotSynchronized), (\"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c\", TimeNotSynchronized), (\"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2\", TimeNotSynchronized), (\"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc\", TimeNotSynchronized), (\"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9\", TimeNotSynchronized), (\"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d\", TimeNotSynchronized), (\"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046\", TimeNotSynchronized), (\"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876\", TimeNotSynchronized), (\"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c\", TimeNotSynchronized)]","latency_us":466518,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:32.70230457Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:472","retry_after":"10.494152653s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"140661b4-14dc-49a6-a219-d42e1a73fe5e\", \"content-length\": \"124\", \"date\": \"Thu, 25 Jul 2024 05:46:32 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"140661b4-14dc-49a6-a219-d42e1a73fe5e\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3f311cd5-ee38-4707-ab8a-9331491f8e7c, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6389ba5c-9d50-4fe4-b1a2-8be3c6861cb0, kind: Service { id: 3f311cd5-ee38-4707-ab8a-9331491f8e7c }, name: Name(\"ntp-3f311cd5-ee38-4707-ab8a-9331491f8e7c\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 235])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4f6a29ee-ca23-4fa8-be69-497060121028, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } } } }, OmicronZoneConfig { id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } } } }, OmicronZoneConfig { id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External } } } }, OmicronZoneConfig { id: b04db43b-dc24-43ce-9d97-32193f88e046, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f04fae05-ec27-43ac-ad4d-ab63c091bf1b, kind: Service { id: b04db43b-dc24-43ce-9d97-32193f88e046 }, name: Name(\"external-dns-b04db43b-dc24-43ce-9d97-32193f88e046\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 154])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 2c3f8453-233f-428f-93cc-0bfefb84929d, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: a6846ef5-8686-4c7e-ae28-d178a4029a85, kind: Service { id: 2c3f8453-233f-428f-93cc-0bfefb84929d }, name: Name(\"external-dns-2c3f8453-233f-428f-93cc-0bfefb84929d\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 164, 190])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: e4efbc31-e513-4290-82f5-c51d8afea0c7, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 735e2401-2ba5-4079-b4a0-7f0184716575, kind: Service { id: e4efbc31-e513-4290-82f5-c51d8afea0c7 }, name: Name(\"nexus-e4efbc31-e513-4290-82f5-c51d8afea0c7\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 154])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 85472347-dfd9-425e-bd98-9b81ebdf3b2b, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: f33554f0-0c2d-4b66-98d0-73365d475c4b, kind: Service { id: 85472347-dfd9-425e-bd98-9b81ebdf3b2b }, name: Name(\"nexus-85472347-dfd9-425e-bd98-9b81ebdf3b2b\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 244])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 221c5e7c-0832-4890-8b0a-f9ba08725876, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 9e8859b7-3d6e-4066-b58c-24fc535c1122, kind: Service { id: 221c5e7c-0832-4890-8b0a-f9ba08725876 }, name: Name(\"nexus-221c5e7c-0832-4890-8b0a-f9ba08725876\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 155])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4b4998f4-10de-49f2-9326-b482137c5fd3, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: a635cc62-7516-4fd9-b490-62d6de386095, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: ac9fa851-e7e1-4295-ac79-d6076526f465, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 1cb96976-2047-44b8-a79c-9fcd74d17bfa, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: a0b9fba0-c330-402f-9bc0-a7510786c98a, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } } } }, OmicronZoneConfig { id: d94d9602-d610-44ce-b9fc-06ded41a1480, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } } } }, OmicronZoneConfig { id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External } } } }, OmicronZoneConfig { id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External } } } }, OmicronZoneConfig { id: dfb1f626-0f55-4385-87de-de59052d98d2, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External } } } }, OmicronZoneConfig { id: adff3617-5320-4e42-8b10-1430a1eb0afc, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:43.226749698Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:434"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:43.227588055Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:43.227597599Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:43.250781713Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:43.250832602Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:43.270849339Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:43.270892418Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4f6a29ee-ca23-4fa8-be69-497060121028, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:43.289755936Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:43.289811119Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4f6a29ee-ca23-4fa8-be69-497060121028, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:43.311104418Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:43.311148097Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:43.329381215Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:43.329428119Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:43.347744761Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:43.347794047Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:43.366931297Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:43.366976982Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b04db43b-dc24-43ce-9d97-32193f88e046, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:43.386727269Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:43.386809837Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: b04db43b-dc24-43ce-9d97-32193f88e046, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2c3f8453-233f-428f-93cc-0bfefb84929d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:43.406068181Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:43.406117157Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2c3f8453-233f-428f-93cc-0bfefb84929d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:43.424882596Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:43.427194141Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a0b9fba0-c330-402f-9bc0-a7510786c98a, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:43.444117903Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:43.44416477Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: a0b9fba0-c330-402f-9bc0-a7510786c98a, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:43.462810435Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:43.462934958Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d94d9602-d610-44ce-b9fc-06ded41a1480, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:43.481487131Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:43.481539258Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d94d9602-d610-44ce-b9fc-06ded41a1480, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:43.499939344Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:43.500056094Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:43.518772356Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:43.518826338Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, dataset_name: DatasetName { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:43.537742195Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:43.537792108Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, dataset_name: DatasetName { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dfb1f626-0f55-4385-87de-de59052d98d2, dataset_name: DatasetName { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:43.555959905Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:43.556016381Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: dfb1f626-0f55-4385-87de-de59052d98d2, dataset_name: DatasetName { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: adff3617-5320-4e42-8b10-1430a1eb0afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:43.574398159Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:43.574453652Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: adff3617-5320-4e42-8b10-1430a1eb0afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, dataset_name: DatasetName { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:43.593079724Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:43.59313575Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, dataset_name: DatasetName { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:43.612766305Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/6d2aa15e-618e-402e-b368-cf27fd8def68/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:43.612831801Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/75c788e1-45ef-4fcf-958d-db07331bef52/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:43.613141521Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:43.613362839Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:43.708438252Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:43.708499068Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:43.708507693Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:43.708514997Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:43.708528666Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:43.708600259Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:43.708607778Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:43.708614839Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:43.708620389Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:43.708625685Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:43.708632557Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:43.708639092Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:43.708644642Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:43.708652602Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:43.708660289Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:43.708666964Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:43.708672158Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:43.708678599Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:43.708688424Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:43.708770411Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"fe18f56b-9776-468e-9bc0-8b26bb2cae19","remote_addr":"[fd00:1122:3344:101::1]:47495","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046\", TimeNotSynchronized), (\"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b\", TimeNotSynchronized), (\"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3\", TimeNotSynchronized), (\"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa\", TimeNotSynchronized), (\"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a\", TimeNotSynchronized), (\"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c\", TimeNotSynchronized), (\"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480\", TimeNotSynchronized), (\"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c\", TimeNotSynchronized), (\"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e\", TimeNotSynchronized), (\"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2\", TimeNotSynchronized), (\"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9\", TimeNotSynchronized), (\"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d\", TimeNotSynchronized), (\"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7\", TimeNotSynchronized), (\"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862\", TimeNotSynchronized), (\"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465\", TimeNotSynchronized), (\"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc\", TimeNotSynchronized), (\"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79\", TimeNotSynchronized), (\"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876\", TimeNotSynchronized), (\"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095\", TimeNotSynchronized)]","latency_us":480518,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:46:43.708958009Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:472","retry_after":"18.50844288s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"fe18f56b-9776-468e-9bc0-8b26bb2cae19\", \"content-length\": \"124\", \"date\": \"Thu, 25 Jul 2024 05:46:43 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"fe18f56b-9776-468e-9bc0-8b26bb2cae19\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:52.276897354Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:45289"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:46:52.276969365Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"e81dcabc-cc6c-42a2-87e4-2c63d45d5cec","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:45289","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":39,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3f311cd5-ee38-4707-ab8a-9331491f8e7c, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6389ba5c-9d50-4fe4-b1a2-8be3c6861cb0, kind: Service { id: 3f311cd5-ee38-4707-ab8a-9331491f8e7c }, name: Name(\"ntp-3f311cd5-ee38-4707-ab8a-9331491f8e7c\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 235])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4f6a29ee-ca23-4fa8-be69-497060121028, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } } } }, OmicronZoneConfig { id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } } } }, OmicronZoneConfig { id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External } } } }, OmicronZoneConfig { id: b04db43b-dc24-43ce-9d97-32193f88e046, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f04fae05-ec27-43ac-ad4d-ab63c091bf1b, kind: Service { id: b04db43b-dc24-43ce-9d97-32193f88e046 }, name: Name(\"external-dns-b04db43b-dc24-43ce-9d97-32193f88e046\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 154])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 2c3f8453-233f-428f-93cc-0bfefb84929d, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: a6846ef5-8686-4c7e-ae28-d178a4029a85, kind: Service { id: 2c3f8453-233f-428f-93cc-0bfefb84929d }, name: Name(\"external-dns-2c3f8453-233f-428f-93cc-0bfefb84929d\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 164, 190])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: e4efbc31-e513-4290-82f5-c51d8afea0c7, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 735e2401-2ba5-4079-b4a0-7f0184716575, kind: Service { id: e4efbc31-e513-4290-82f5-c51d8afea0c7 }, name: Name(\"nexus-e4efbc31-e513-4290-82f5-c51d8afea0c7\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 154])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 85472347-dfd9-425e-bd98-9b81ebdf3b2b, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: f33554f0-0c2d-4b66-98d0-73365d475c4b, kind: Service { id: 85472347-dfd9-425e-bd98-9b81ebdf3b2b }, name: Name(\"nexus-85472347-dfd9-425e-bd98-9b81ebdf3b2b\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 244])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 221c5e7c-0832-4890-8b0a-f9ba08725876, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 9e8859b7-3d6e-4066-b58c-24fc535c1122, kind: Service { id: 221c5e7c-0832-4890-8b0a-f9ba08725876 }, name: Name(\"nexus-221c5e7c-0832-4890-8b0a-f9ba08725876\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 155])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4b4998f4-10de-49f2-9326-b482137c5fd3, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: a635cc62-7516-4fd9-b490-62d6de386095, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: ac9fa851-e7e1-4295-ac79-d6076526f465, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 1cb96976-2047-44b8-a79c-9fcd74d17bfa, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: a0b9fba0-c330-402f-9bc0-a7510786c98a, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } } } }, OmicronZoneConfig { id: d94d9602-d610-44ce-b9fc-06ded41a1480, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } } } }, OmicronZoneConfig { id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External } } } }, OmicronZoneConfig { id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External } } } }, OmicronZoneConfig { id: dfb1f626-0f55-4385-87de-de59052d98d2, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External } } } }, OmicronZoneConfig { id: adff3617-5320-4e42-8b10-1430a1eb0afc, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:02.269328316Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:434"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:02.269564656Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:02.269577256Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:02.292322792Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:02.292393453Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:02.310451213Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:02.310504632Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4f6a29ee-ca23-4fa8-be69-497060121028, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:02.329878522Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:02.329992054Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4f6a29ee-ca23-4fa8-be69-497060121028, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:02.348110054Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:02.348169325Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:02.366263341Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:02.366308492Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:02.38389173Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:02.383935588Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:02.402295258Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:02.402352484Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b04db43b-dc24-43ce-9d97-32193f88e046, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:02.420772843Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:02.420848764Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: b04db43b-dc24-43ce-9d97-32193f88e046, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2c3f8453-233f-428f-93cc-0bfefb84929d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:02.439051426Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:02.439109835Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2c3f8453-233f-428f-93cc-0bfefb84929d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:02.457636233Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:02.457695381Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a0b9fba0-c330-402f-9bc0-a7510786c98a, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:02.476614397Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:02.476670293Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: a0b9fba0-c330-402f-9bc0-a7510786c98a, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:02.497045558Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:02.497093456Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d94d9602-d610-44ce-b9fc-06ded41a1480, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:02.514800328Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:02.51485865Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d94d9602-d610-44ce-b9fc-06ded41a1480, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:02.533404218Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:02.533450081Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:02.551182165Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:02.551234329Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, dataset_name: DatasetName { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:02.570799841Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:02.570863714Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, dataset_name: DatasetName { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dfb1f626-0f55-4385-87de-de59052d98d2, dataset_name: DatasetName { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:02.59107317Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:02.591129215Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: dfb1f626-0f55-4385-87de-de59052d98d2, dataset_name: DatasetName { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: adff3617-5320-4e42-8b10-1430a1eb0afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:02.608749798Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:02.608797583Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: adff3617-5320-4e42-8b10-1430a1eb0afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, dataset_name: DatasetName { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:02.627589891Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:02.627650379Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, dataset_name: DatasetName { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:02.644898018Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/6d2aa15e-618e-402e-b368-cf27fd8def68/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:02.644966289Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/75c788e1-45ef-4fcf-958d-db07331bef52/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:02.645294824Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:02.645484317Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:02.738319083Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:02.738397563Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:02.738414842Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:02.738421901Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:02.738429974Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:02.738447299Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:02.738455389Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:02.738468787Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:02.738477543Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:02.738485559Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:02.738493884Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:02.738507582Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:02.738519376Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:02.738524138Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:02.738535079Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:02.73853957Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:02.73854723Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:02.738552011Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:02.738558142Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:02.738619663Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"b979f377-2c30-48db-bce9-c83b90f10cd5","remote_addr":"[fd00:1122:3344:101::1]:47495","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c\", TimeNotSynchronized), (\"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a\", TimeNotSynchronized), (\"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480\", TimeNotSynchronized), (\"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2\", TimeNotSynchronized), (\"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc\", TimeNotSynchronized), (\"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046\", TimeNotSynchronized), (\"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862\", TimeNotSynchronized), (\"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9\", TimeNotSynchronized), (\"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3\", TimeNotSynchronized), (\"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c\", TimeNotSynchronized), (\"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79\", TimeNotSynchronized), (\"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e\", TimeNotSynchronized), (\"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465\", TimeNotSynchronized), (\"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d\", TimeNotSynchronized), (\"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7\", TimeNotSynchronized), (\"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876\", TimeNotSynchronized), (\"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095\", TimeNotSynchronized), (\"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b\", TimeNotSynchronized), (\"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa\", TimeNotSynchronized)]","latency_us":467963,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:02.738837411Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:472","retry_after":"8.905614665s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b979f377-2c30-48db-bce9-c83b90f10cd5\", \"content-length\": \"124\", \"date\": \"Thu, 25 Jul 2024 05:47:02 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b979f377-2c30-48db-bce9-c83b90f10cd5\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:09.063891178Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","component":"MetricsManager","sled_id":"8fa4bc78-1026-41f0-b09d-c67224071376","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"40.633740955s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3f311cd5-ee38-4707-ab8a-9331491f8e7c, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6389ba5c-9d50-4fe4-b1a2-8be3c6861cb0, kind: Service { id: 3f311cd5-ee38-4707-ab8a-9331491f8e7c }, name: Name(\"ntp-3f311cd5-ee38-4707-ab8a-9331491f8e7c\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 235])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4f6a29ee-ca23-4fa8-be69-497060121028, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } } } }, OmicronZoneConfig { id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } } } }, OmicronZoneConfig { id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External } } } }, OmicronZoneConfig { id: b04db43b-dc24-43ce-9d97-32193f88e046, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f04fae05-ec27-43ac-ad4d-ab63c091bf1b, kind: Service { id: b04db43b-dc24-43ce-9d97-32193f88e046 }, name: Name(\"external-dns-b04db43b-dc24-43ce-9d97-32193f88e046\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 154])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 2c3f8453-233f-428f-93cc-0bfefb84929d, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: a6846ef5-8686-4c7e-ae28-d178a4029a85, kind: Service { id: 2c3f8453-233f-428f-93cc-0bfefb84929d }, name: Name(\"external-dns-2c3f8453-233f-428f-93cc-0bfefb84929d\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 164, 190])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: e4efbc31-e513-4290-82f5-c51d8afea0c7, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 735e2401-2ba5-4079-b4a0-7f0184716575, kind: Service { id: e4efbc31-e513-4290-82f5-c51d8afea0c7 }, name: Name(\"nexus-e4efbc31-e513-4290-82f5-c51d8afea0c7\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 154])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 85472347-dfd9-425e-bd98-9b81ebdf3b2b, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: f33554f0-0c2d-4b66-98d0-73365d475c4b, kind: Service { id: 85472347-dfd9-425e-bd98-9b81ebdf3b2b }, name: Name(\"nexus-85472347-dfd9-425e-bd98-9b81ebdf3b2b\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 244])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 221c5e7c-0832-4890-8b0a-f9ba08725876, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 9e8859b7-3d6e-4066-b58c-24fc535c1122, kind: Service { id: 221c5e7c-0832-4890-8b0a-f9ba08725876 }, name: Name(\"nexus-221c5e7c-0832-4890-8b0a-f9ba08725876\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 155])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4b4998f4-10de-49f2-9326-b482137c5fd3, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: a635cc62-7516-4fd9-b490-62d6de386095, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: ac9fa851-e7e1-4295-ac79-d6076526f465, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 1cb96976-2047-44b8-a79c-9fcd74d17bfa, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: a0b9fba0-c330-402f-9bc0-a7510786c98a, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } } } }, OmicronZoneConfig { id: d94d9602-d610-44ce-b9fc-06ded41a1480, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } } } }, OmicronZoneConfig { id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External } } } }, OmicronZoneConfig { id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External } } } }, OmicronZoneConfig { id: dfb1f626-0f55-4385-87de-de59052d98d2, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External } } } }, OmicronZoneConfig { id: adff3617-5320-4e42-8b10-1430a1eb0afc, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:11.669813504Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:434"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:11.670407501Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:11.670426768Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:11.693105536Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:11.693157577Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:11.713456679Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:11.713511411Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4f6a29ee-ca23-4fa8-be69-497060121028, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:11.733456901Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:11.733530253Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4f6a29ee-ca23-4fa8-be69-497060121028, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:11.752661493Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:11.755064299Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:11.770784661Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:11.770835719Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:11.788997977Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:11.789064597Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:11.806428049Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:11.806474166Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b04db43b-dc24-43ce-9d97-32193f88e046, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:11.825186223Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:11.825232041Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: b04db43b-dc24-43ce-9d97-32193f88e046, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2c3f8453-233f-428f-93cc-0bfefb84929d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:11.843211391Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:11.843259937Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2c3f8453-233f-428f-93cc-0bfefb84929d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:11.861918597Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:11.861967751Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a0b9fba0-c330-402f-9bc0-a7510786c98a, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:11.880019971Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:11.880069267Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: a0b9fba0-c330-402f-9bc0-a7510786c98a, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:11.898640322Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:11.898693864Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d94d9602-d610-44ce-b9fc-06ded41a1480, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:11.91647949Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:11.916524435Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d94d9602-d610-44ce-b9fc-06ded41a1480, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:11.934992907Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:11.935034252Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:11.955242139Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:11.955297387Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, dataset_name: DatasetName { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:11.974927629Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:11.974982109Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, dataset_name: DatasetName { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dfb1f626-0f55-4385-87de-de59052d98d2, dataset_name: DatasetName { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:11.993642213Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:11.993691499Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: dfb1f626-0f55-4385-87de-de59052d98d2, dataset_name: DatasetName { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: adff3617-5320-4e42-8b10-1430a1eb0afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:12.012524336Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:12.012596948Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: adff3617-5320-4e42-8b10-1430a1eb0afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, dataset_name: DatasetName { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:12.034654716Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:12.034713367Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, dataset_name: DatasetName { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:12.053899645Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/6d2aa15e-618e-402e-b368-cf27fd8def68/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:12.053959216Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/75c788e1-45ef-4fcf-958d-db07331bef52/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:12.054179739Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:12.054380041Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:12.150596973Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:12.150671206Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:12.150684088Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:12.150691231Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:12.150696782Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:12.150703748Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:12.15072022Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:12.15072742Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:12.150734555Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:12.150739721Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:12.150746611Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:12.150751655Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:12.150758349Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:12.150764875Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:12.150768681Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:12.150773772Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:12.150782247Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:12.150791678Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:12.150796769Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3066","zone":"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:12.150852017Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"0b7709c4-0e9c-4f77-a1f8-8c61e67f03b3","remote_addr":"[fd00:1122:3344:101::1]:47495","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79\", TimeNotSynchronized), (\"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d\", TimeNotSynchronized), (\"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2\", TimeNotSynchronized), (\"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa\", TimeNotSynchronized), (\"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7\", TimeNotSynchronized), (\"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b\", TimeNotSynchronized), (\"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876\", TimeNotSynchronized), (\"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862\", TimeNotSynchronized), (\"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465\", TimeNotSynchronized), (\"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095\", TimeNotSynchronized), (\"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c\", TimeNotSynchronized), (\"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046\", TimeNotSynchronized), (\"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480\", TimeNotSynchronized), (\"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e\", TimeNotSynchronized), (\"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc\", TimeNotSynchronized), (\"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c\", TimeNotSynchronized), (\"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9\", TimeNotSynchronized), (\"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a\", TimeNotSynchronized), (\"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3\", TimeNotSynchronized)]","latency_us":479535,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:12.151035893Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:472","retry_after":"23.63926763s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"0b7709c4-0e9c-4f77-a1f8-8c61e67f03b3\", \"content-length\": \"124\", \"date\": \"Thu, 25 Jul 2024 05:47:11 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"0b7709c4-0e9c-4f77-a1f8-8c61e67f03b3\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:22.392144485Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:60831"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:22.392264968Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"4933053c-1f7e-4c29-9717-86f054558d6d","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:60831","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":35,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 3f311cd5-ee38-4707-ab8a-9331491f8e7c, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6389ba5c-9d50-4fe4-b1a2-8be3c6861cb0, kind: Service { id: 3f311cd5-ee38-4707-ab8a-9331491f8e7c }, name: Name(\"ntp-3f311cd5-ee38-4707-ab8a-9331491f8e7c\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 246, 235])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 4f6a29ee-ca23-4fa8-be69-497060121028, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } } } }, OmicronZoneConfig { id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } } } }, OmicronZoneConfig { id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External } } } }, OmicronZoneConfig { id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External } } } }, OmicronZoneConfig { id: b04db43b-dc24-43ce-9d97-32193f88e046, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: f04fae05-ec27-43ac-ad4d-ab63c091bf1b, kind: Service { id: b04db43b-dc24-43ce-9d97-32193f88e046 }, name: Name(\"external-dns-b04db43b-dc24-43ce-9d97-32193f88e046\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 154])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 2c3f8453-233f-428f-93cc-0bfefb84929d, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: a6846ef5-8686-4c7e-ae28-d178a4029a85, kind: Service { id: 2c3f8453-233f-428f-93cc-0bfefb84929d }, name: Name(\"external-dns-2c3f8453-233f-428f-93cc-0bfefb84929d\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 164, 190])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: e4efbc31-e513-4290-82f5-c51d8afea0c7, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 735e2401-2ba5-4079-b4a0-7f0184716575, kind: Service { id: e4efbc31-e513-4290-82f5-c51d8afea0c7 }, name: Name(\"nexus-e4efbc31-e513-4290-82f5-c51d8afea0c7\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 154])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 85472347-dfd9-425e-bd98-9b81ebdf3b2b, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: f33554f0-0c2d-4b66-98d0-73365d475c4b, kind: Service { id: 85472347-dfd9-425e-bd98-9b81ebdf3b2b }, name: Name(\"nexus-85472347-dfd9-425e-bd98-9b81ebdf3b2b\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 244])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 221c5e7c-0832-4890-8b0a-f9ba08725876, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 9e8859b7-3d6e-4066-b58c-24fc535c1122, kind: Service { id: 221c5e7c-0832-4890-8b0a-f9ba08725876 }, name: Name(\"nexus-221c5e7c-0832-4890-8b0a-f9ba08725876\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 155])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4b4998f4-10de-49f2-9326-b482137c5fd3, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: a635cc62-7516-4fd9-b490-62d6de386095, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: ac9fa851-e7e1-4295-ac79-d6076526f465, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 1cb96976-2047-44b8-a79c-9fcd74d17bfa, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: a0b9fba0-c330-402f-9bc0-a7510786c98a, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External } } } }, OmicronZoneConfig { id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External } } } }, OmicronZoneConfig { id: d94d9602-d610-44ce-b9fc-06ded41a1480, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External } } } }, OmicronZoneConfig { id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External } } } }, OmicronZoneConfig { id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External } } } }, OmicronZoneConfig { id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External } } } }, OmicronZoneConfig { id: dfb1f626-0f55-4385-87de-de59052d98d2, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External } } } }, OmicronZoneConfig { id: adff3617-5320-4e42-8b10-1430a1eb0afc, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External } } } }, OmicronZoneConfig { id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:35.85636956Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:434"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:35.856818642Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:35.856829057Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: aa38c6ce-70ac-4e2b-a383-8201d1e086e8, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:35.879588686Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:35.879656806Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 0ee9b880-7690-4963-bc6b-3dbf4d39892d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:35.899024954Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:35.89907873Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 23e793cc-4c0e-4aca-92a8-2b30f6dafa5a, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4f6a29ee-ca23-4fa8-be69-497060121028, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:35.916866403Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:35.916928261Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4f6a29ee-ca23-4fa8-be69-497060121028, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:35.934593096Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:35.934661337Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 3b0743b7-2816-4de6-a40a-d863e46d23f8, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:35.954087855Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:35.954144285Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d835fd87-0881-47dd-8e5b-ebb01fd7b0ce, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:35.972833404Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:35.972884462Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 33444b97-3e72-47f7-a319-cbd1ca92b6c8, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:35.9916251Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:35.991682965Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4553d8e1-a7ca-4d20-b1a4-394640a4ad4f, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b04db43b-dc24-43ce-9d97-32193f88e046, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:36.010964083Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:36.011016659Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: b04db43b-dc24-43ce-9d97-32193f88e046, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2c3f8453-233f-428f-93cc-0bfefb84929d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:36.030922412Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:36.030990017Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2c3f8453-233f-428f-93cc-0bfefb84929d, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:36.051859314Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:36.051923083Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 4ba7467e-cf15-4665-9aba-eeedcaaaa862, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: a0b9fba0-c330-402f-9bc0-a7510786c98a, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:36.071498545Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:36.07154544Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: a0b9fba0-c330-402f-9bc0-a7510786c98a, dataset_name: DatasetName { pool_name: ZpoolName { id: dd1c81eb-5ddf-42bb-acfe-348d2d141fca (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:36.091137725Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:36.091189974Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: ee1bae54-08c6-4724-bc97-b5188feb1b8c, dataset_name: DatasetName { pool_name: ZpoolName { id: 59d3cc5a-40b7-44b8-b2af-cfc21755b8bf (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d94d9602-d610-44ce-b9fc-06ded41a1480, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:36.11015938Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:36.110203415Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: d94d9602-d610-44ce-b9fc-06ded41a1480, dataset_name: DatasetName { pool_name: ZpoolName { id: e78b4f40-df20-4bcf-bd8a-b80afa109d70 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:36.128544027Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:36.128600025Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c, dataset_name: DatasetName { pool_name: ZpoolName { id: 63a12260-6721-4f51-85d5-262aedc90628 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:36.146549213Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:36.14660313Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 7f7f1d81-12f9-4239-8e97-9582c9218b79, dataset_name: DatasetName { pool_name: ZpoolName { id: 587239d7-a145-4bb7-b24a-aff014195deb (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, dataset_name: DatasetName { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:36.164328282Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:36.164378289Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 2e3ea66c-7cbd-4cae-aca1-5ebcb142282e, dataset_name: DatasetName { pool_name: ZpoolName { id: 50e63493-6b9c-42de-846b-797a312b5fbc (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: dfb1f626-0f55-4385-87de-de59052d98d2, dataset_name: DatasetName { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:36.182257353Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:36.182307023Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: dfb1f626-0f55-4385-87de-de59052d98d2, dataset_name: DatasetName { pool_name: ZpoolName { id: fd75587e-5d72-48df-a44e-8602b2d61469 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: adff3617-5320-4e42-8b10-1430a1eb0afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:36.200502991Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:36.200626578Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: adff3617-5320-4e42-8b10-1430a1eb0afc, dataset_name: DatasetName { pool_name: ZpoolName { id: 20d995c8-4908-4380-8438-c04a6e3adea2 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, dataset_name: DatasetName { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:36.218755426Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:36.218807975Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:771","request":"NewFilesystemRequest { dataset_id: 1f722dd3-3360-4e20-99bf-1de5eb86ffd9, dataset_name: DatasetName { pool_name: ZpoolName { id: e43689a1-378e-4be8-9661-3bc522647c6d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:36.236928431Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/6d2aa15e-618e-402e-b368-cf27fd8def68/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:36.236982553Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/75c788e1-45ef-4fcf-958d-db07331bef52/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:36.237277545Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:36.237427895Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:36.348772371Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:258"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:36.349051727Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:36.360663508Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:258"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:36.360814534Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:36.395598038Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:258"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:36.395784455Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:36.395810677Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:36.395841662Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:36.420381157Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:258"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:36.420630501Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:36.432079844Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:36.57380533Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:258"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:36.574035529Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:36.574070246Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:36.600774459Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:36.600839786Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:36.600853445Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:36.600859107Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:36.600864367Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Configuring new Omicron zone: oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:36.600997383Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:36.606525927Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:36.612535791Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:36.617856741Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:36.623220974Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:36.629318621Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:36.635605528Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:36.779479693Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:36.784788264Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:36.789913186Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:36.79453584Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:36.799330498Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:36.804590293Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:36.809461507Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:36.814569512Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:36.819576195Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:36.825833161Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:36.831524125Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:36.83768178Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:36.842965819Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:36.848804177Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:36.891779277Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:275","addr":"fd00:1122:3344:101::2"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:36.892007761Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:36.899868058Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:36.906589677Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:36.912766311Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:36.920632541Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:36.929016893Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:36.936754019Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:37.040311514Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:275","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:37.163903875Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:275","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:37.224634409Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:275","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:37.305805109Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:275","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:37.30783948Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:300"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:37.307859384Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:192","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:37.360087498Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:426","port":"Port { inner: PortInner(PortData { name: \"opte1\", ip: 172.30.2.6, mac: MacAddr6([168, 64, 37, 255, 150, 244]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:37.360134252Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:1207","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:37.363254035Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:300"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:37.363272214Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:192","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:37.420077513Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:426","port":"Port { inner: PortInner(PortData { name: \"opte2\", ip: 172.30.1.5, mac: MacAddr6([168, 64, 37, 255, 194, 154]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:37.420107767Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:1207","zone_type":"external_dns"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:37.423573839Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:300"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:37.423587358Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:192","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:37.48265271Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:426","port":"Port { inner: PortInner(PortData { name: \"opte3\", ip: 172.30.1.6, mac: MacAddr6([168, 64, 37, 255, 164, 190]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:37.482683714Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:1207","zone_type":"external_dns"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:37.484365821Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:300"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:37.484379658Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:192","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:37.544644069Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:426","port":"Port { inner: PortInner(PortData { name: \"opte4\", ip: 172.30.2.5, mac: MacAddr6([168, 64, 37, 255, 205, 154]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:37.544675664Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:1207","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:37.546460119Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:300"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:37.546474069Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:192","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:37.599875098Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:426","port":"Port { inner: PortInner(PortData { name: \"opte5\", ip: 172.30.2.7, mac: MacAddr6([168, 64, 37, 255, 245, 155]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:37.599907556Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:1207","zone_type":"nexus"} {"msg":"no nat entry found for: 10.151.1.164","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:37.600690732Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.164","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:37.60070261Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"no nat entry found for: 10.151.1.167","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:37.600875846Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.167","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:37.600884611Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"no nat entry found for: 10.151.1.168","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:37.603072152Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.168","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:37.603083384Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"no nat entry found for: 10.151.1.165","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:37.603218312Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.165","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:37.6032299Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"no nat entry found for: 10.151.1.166","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:37.603405933Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.166","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:37.603416199Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"creation of nat entry successful for: 10.151.1.164","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:37.605418799Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:37.607610152Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"creation of nat entry successful for: 10.151.1.168","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:37.670077888Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:37.674182795Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"creation of nat entry successful for: 10.151.1.165","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:37.674305395Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"creation of nat entry successful for: 10.151.1.167","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:37.679127365Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"creation of nat entry successful for: 10.151.1.166","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:37.682827752Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:37.684823691Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:37.684841336Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:37.908248043Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Configuring new Omicron zone: oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:38.415227665Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:38.471192934Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:38.640553595Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:38.676508039Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:38.890955092Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:38.970963952Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:39.010280735Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:39.062911615Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:39.076516948Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:39.08671527Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:41.199967181Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:41.200367401Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"Profile for oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:41.227619078Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79 zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:41.229678727Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"Profile for oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:41.246510716Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:41.247419318Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:41.247658624Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"Booting oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480 zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:41.259216128Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"Profile for oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:41.273518018Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:41.273919477Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"Profile for oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:41.289061497Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:41.289491987Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"Profile for oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:41.315620017Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095 zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:41.315869243Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"Profile for oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:41.327418788Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9 zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:41.327872198Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"Profile for oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:41.42225192Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2 zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:41.433043486Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"Profile for oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:41.500262412Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3 zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:41.500460667Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"wait for service Some(\"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c\") failed: Property not found. retry in 37.426328ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:41.730245583Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9\") failed: Property not found. retry in 54.126915ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:41.749665424Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480\") failed: Property not found. retry in 33.224664ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:41.767443126Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc\") failed: Property not found. retry in 50.529394ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:41.79245254Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c\") failed: Property not found. retry in 77.409831ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:48.781508281Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79\") failed: Property not found. retry in 42.044072ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:49.099893682Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095\") failed: Property not found. retry in 70.216509ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:49.11411279Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c\") failed: Property not found. retry in 52.884266ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:49.123485272Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e\") failed: Property not found. retry in 25.355646ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:49.144175493Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c\") failed: Property not found. retry in 241.507764ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:49.443239025Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9\") failed: Property not found. retry in 145.616899ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:49.490399895Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480\") failed: Property not found. retry in 97.309407ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:50.536851984Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:50.856194497Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79\") failed: Property not found. retry in 51.278324ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:54.731044749Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2\") failed: Property not found. retry in 49.792323ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:54.77701533Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc\") failed: Property not found. retry in 86.940724ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:54.845805944Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095\") failed: Property not found. retry in 95.684907ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:54.875612762Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:54.894617223Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:52740"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:54.905548182Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"431a883f-3a90-4063-a4b7-cd143ff04f74","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:52740","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":7150,"response_code":"200"} {"msg":"wait for service Some(\"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c\") failed: Property not found. retry in 125.800962ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:54.912355994Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3\") failed: Property not found. retry in 65.671107ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:54.949534617Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e\") failed: Property not found. retry in 126.539011ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:54.990864558Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c\") failed: Property not found. retry in 265.635286ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:55.023343819Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:55.024492927Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9\") failed: Property not found. retry in 140.196785ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:55.048998849Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480\") failed: Property not found. retry in 104.642786ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:55.066929385Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Booting oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:55.073751377Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"wait for service Some(\"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79\") failed: Property not found. retry in 180.180835ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:55.096343253Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2\") failed: Property not found. retry in 89.701615ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:55.111345539Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc\") failed: Property not found. retry in 208.166526ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:55.127645296Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095\") failed: Property not found. retry in 227.55301ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:55.143832491Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c\") failed: Property not found. retry in 161.885465ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:55.15727025Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3\") failed: Property not found. retry in 84.941899ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:55.170078844Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e\") failed: Property not found. retry in 131.728238ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:55.182384906Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Booting oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:55.18253402Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"wait for service Some(\"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9\") failed: Property not found. retry in 575.016246ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:55.210910064Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480\") failed: Property not found. retry in 519.11869ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:55.22210897Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2\") failed: Property not found. retry in 238.407015ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:55.234234407Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:55.235289023Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:55.237175857Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465 zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:55.241103001Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"Booting oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:55.251740223Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"wait for service Some(\"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3\") failed: Property not found. retry in 118.069082ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:55.272463985Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79\") failed: Property not found. retry in 326.503453ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:55.290440992Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c\") failed: Property not found. retry in 503.021503ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:55.303212165Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e\") failed: Property not found. retry in 402.413252ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:55.326701093Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c\") failed: Property not found. retry in 318.710239ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:55.33919846Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc\") failed: Property not found. retry in 417.770481ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:55.350599921Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095\") failed: Property not found. retry in 221.787551ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:55.3867699Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3\") failed: Property not found. retry in 292.956591ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:55.403370474Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:55.419930527Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:55.4204611Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"Profile for oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:55.430489288Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d\") failed: Property not found. retry in 37.296843ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:55.441501229Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Booting oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7 zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:55.441628523Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"wait for service Some(\"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a\") failed: Property not found. retry in 53.304346ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:55.474739119Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2\") failed: Property not found. retry in 431.749051ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:55.485778421Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d\") failed: Property not found. retry in 88.66187ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:55.496093276Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa\") failed: Property not found. retry in 54.785529ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:55.512142067Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:55.51355361Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465\") failed: Property not found. retry in 57.268277ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:57.111624598Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Booting oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046 zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:57.144309144Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"wait for service Some(\"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc\") failed: Property not found. retry in 665.950737ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:57.186650109Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095\") failed: Property not found. retry in 620.479039ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:57.219477049Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c\") failed: Property not found. retry in 1.100537669s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:57.234374393Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3\") failed: Property not found. retry in 903.640511ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:57.250986977Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465\") failed: Property not found. retry in 138.082542ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:57.274149907Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa\") failed: Property not found. retry in 70.413134ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:57.283926619Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e\") failed: Property not found. retry in 795.102851ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:57.295090974Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c\") failed: Property not found. retry in 761.387724ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:57.305501811Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d\") failed: Property not found. retry in 135.910967ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:57.316824771Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9\") failed: Property not found. retry in 463.039605ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:57.32901408Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b\") failed: Property not found. retry in 29.527185ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:57.341334882Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480\") failed: Property not found. retry in 703.862656ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:57.352445506Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a\") failed: Property not found. retry in 55.565974ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:57.3677478Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79\") failed: Property not found. retry in 1.166835065s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:57.378310971Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7\") failed: Property not found. retry in 48.044075ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:57.38881287Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2\") failed: Property not found. retry in 823.606654ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:57.399416202Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa\") failed: Property not found. retry in 114.450589ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:57.409599334Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b\") failed: Property not found. retry in 135.281344ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:57.420641466Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046\") failed: Property not found. retry in 61.128055ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:57.430271785Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465\") failed: Property not found. retry in 289.35517ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:57.438417618Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a\") failed: Property not found. retry in 180.054723ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:57.449066531Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7\") failed: Property not found. retry in 66.844982ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:57.461384313Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d\") failed: Property not found. retry in 565.940208ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:57.471678668Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046\") failed: Property not found. retry in 69.37724ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:57.503035682Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa\") failed: Property not found. retry in 584.99465ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:57.536517188Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7\") failed: Property not found. retry in 291.532203ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:57.544233321Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b\") failed: Property not found. retry in 198.510149ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:57.567038562Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046\") failed: Property not found. retry in 286.656199ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:57.586987435Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a\") failed: Property not found. retry in 324.083036ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:57.650103144Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465\") failed: Property not found. retry in 446.780388ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:57.740590783Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b\") failed: Property not found. retry in 530.244439ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:57.77794845Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9\") failed: Property not found. retry in 1.351110515s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:57.802297098Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7\") failed: Property not found. retry in 268.797536ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:57.846810875Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095\") failed: Property not found. retry in 731.055593ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:57.856647409Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc\") failed: Property not found. retry in 1.377519051s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:57.866074073Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046\") failed: Property not found. retry in 256.241788ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:57.884314226Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a\") failed: Property not found. retry in 1.079940506s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:57.986214416Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:58.032201018Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876 zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:47:58.03228486Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"wait for service Some(\"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d\") failed: Property not found. retry in 774.87831ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:58.054716662Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480\") failed: Property not found. retry in 677.574285ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:58.072935325Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c\") failed: Property not found. retry in 1.389192488s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:58.087085821Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e\") failed: Property not found. retry in 1.476488727s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:58.101828361Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7\") failed: Property not found. retry in 1.087282128s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:58.127638234Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa\") failed: Property not found. retry in 1.002843619s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:58.136544916Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046\") failed: Property not found. retry in 1.188552694s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:58.15103065Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3\") failed: Property not found. retry in 760.161627ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:58.166828865Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465\") failed: Property not found. retry in 752.772191ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:58.198548578Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2\") failed: Property not found. retry in 593.833015ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:58.236031198Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876\") failed: Property not found. retry in 63.719909ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:58.26011034Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b\") failed: Property not found. retry in 626.855082ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:58.320297619Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876\") failed: Property not found. retry in 145.846146ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:58.333902702Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c\") failed: Property not found. retry in 905.236348ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:58.36244926Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876\") failed: Property not found. retry in 163.363908ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:58.49467872Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79\") failed: Property not found. retry in 1.177290733s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:58.557289556Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095\") failed: Property not found. retry in 603.629053ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:58.602848689Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876\") failed: Property not found. retry in 569.340709ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:58.674571802Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480\") failed: Property not found. retry in 880.187151ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:58.764928737Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d\") failed: Property not found. retry in 1.09792147s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:58.843588015Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2\") failed: Property not found. retry in 833.595025ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:58.854389082Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3\") failed: Property not found. retry in 1.227164237s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:58.939254077Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b\") failed: Property not found. retry in 615.973401ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:58.962439508Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465\") failed: Property not found. retry in 1.192478695s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:58.974135645Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a\") failed: Property not found. retry in 866.172582ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:59.083369679Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa\") failed: Property not found. retry in 610.016509ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:59.154825976Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9\") failed: Property not found. retry in 1.381868421s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:59.168398339Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095\") failed: Property not found. retry in 946.164709ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:59.221072569Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7\") failed: Property not found. retry in 879.632144ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:59.236855974Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc\") failed: Property not found. retry in 1.206140776s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:59.259456921Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876\") failed: Property not found. retry in 959.495122ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:59.276485275Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c\") failed: Property not found. retry in 794.853114ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:59.296306466Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046\") failed: Property not found. retry in 670.679984ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:59.354947538Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c\") failed: Property not found. retry in 698.874985ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:59.493306424Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b\") failed: Property not found. retry in 647.971118ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:59.598338848Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e\") failed: Property not found. retry in 1.095156255s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:59.618543087Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480\") failed: Property not found. retry in 569.621801ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:59.660246848Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2\") failed: Property not found. retry in 1.424332475s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:59.709089078Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79\") failed: Property not found. retry in 1.457034465s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:59.754290041Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa\") failed: Property not found. retry in 1.398853443s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:59.782319717Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d\") failed: Property not found. retry in 1.411180647s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:59.958598693Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a\") failed: Property not found. retry in 674.383159ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:47:59.973599949Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046\") failed: Property not found. retry in 1.332190553s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:00.042512945Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c\") failed: Property not found. retry in 742.420306ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:00.106883964Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7\") failed: Property not found. retry in 839.973084ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:00.131857377Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3\") failed: Property not found. retry in 1.120226528s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:00.179604221Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465\") failed: Property not found. retry in 1.140134864s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:00.19305025Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c\") failed: Property not found. retry in 582.304513ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:00.206389657Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095\") failed: Property not found. retry in 825.450297ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:00.217083391Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480\") failed: Property not found. retry in 1.008307029s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:00.242121576Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876\") failed: Property not found. retry in 977.954121ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:00.254278034Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b\") failed: Property not found. retry in 731.029008ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:00.26841448Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc\") failed: Property not found. retry in 1.299031098s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:00.480266601Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9\") failed: Property not found. retry in 918.154749ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:00.566078928Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a\") failed: Property not found. retry in 1.089652177s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:00.667164158Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e\") failed: Property not found. retry in 681.033618ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:00.73294122Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c\") failed: Property not found. retry in 1.418942366s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:00.802687876Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c\") failed: Property not found. retry in 808.892026ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:00.868337165Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7\") failed: Property not found. retry in 649.61465ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:00.987782862Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b\") failed: Property not found. retry in 1.068968292s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:01.016899083Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095\") failed: Property not found. retry in 1.420865688s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:01.055472219Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2\") failed: Property not found. retry in 1.480647827s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:01.147925535Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa\") failed: Property not found. retry in 561.875433ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:01.193875336Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79\") failed: Property not found. retry in 627.010881ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:01.224529254Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876\") failed: Property not found. retry in 1.362808117s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:01.245286647Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480\") failed: Property not found. retry in 810.424128ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:01.264210866Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3\") failed: Property not found. retry in 1.149008714s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:01.310538076Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465\") failed: Property not found. retry in 520.589744ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:01.344881722Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d\") failed: Property not found. retry in 1.323923045s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:01.383019927Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046\") failed: Property not found. retry in 1.450370551s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:01.395171806Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e\") failed: Property not found. retry in 1.476939112s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:01.425285261Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9\") failed: Property not found. retry in 773.808732ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:01.498460389Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7\") failed: Property not found. retry in 1.067806629s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:01.649056894Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c\") failed: Property not found. retry in 1.497117333s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:01.692416554Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:01.721379422Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862 zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:01.729461344Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa\") failed: Property not found. retry in 628.312703ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:01.769835833Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a\") failed: Property not found. retry in 594.785266ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:01.780550227Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc\") failed: Property not found. retry in 545.382437ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:01.789813367Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79\") failed: Property not found. retry in 1.142089643s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:01.863282472Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465\") failed: Property not found. retry in 530.970014ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:01.877431088Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862\") failed: Property not found. retry in 68.156145ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:01.975849085Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862\") failed: Property not found. retry in 118.316144ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:02.056280255Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480\") failed: Property not found. retry in 702.761469ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:02.087299761Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b\") failed: Property not found. retry in 1.311457962s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:02.09947993Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862\") failed: Property not found. retry in 195.672498ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:02.211663134Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c\") failed: Property not found. retry in 1.330497282s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:02.234397044Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9\") failed: Property not found. retry in 1.239252337s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:02.289273088Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc\") failed: Property not found. retry in 1.224978233s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:02.345636086Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a\") failed: Property not found. retry in 830.877686ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:02.384752925Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa\") failed: Property not found. retry in 806.312745ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:02.406622684Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862\") failed: Property not found. retry in 509.376192ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:02.416720004Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465\") failed: Property not found. retry in 1.0337136s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:02.424476858Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3\") failed: Property not found. retry in 1.131608349s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:02.467671134Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095\") failed: Property not found. retry in 1.260962384s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:02.48476765Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876\") failed: Property not found. retry in 1.450966256s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:02.618171827Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2\") failed: Property not found. retry in 566.117525ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:02.637597269Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d\") failed: Property not found. retry in 1.151011803s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:02.716284507Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7\") failed: Property not found. retry in 833.8049ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:02.725116797Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480\") failed: Property not found. retry in 1.145716971s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:02.799094314Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046\") failed: Property not found. retry in 970.076581ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:02.855348239Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e\") failed: Property not found. retry in 1.19940143s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:02.912274301Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862\") failed: Property not found. retry in 803.626666ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:02.935709848Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79\") failed: Property not found. retry in 1.139490235s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:03.016106017Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c\") failed: Property not found. retry in 982.621469ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:03.201300145Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2\") failed: Property not found. retry in 956.239748ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:03.213946685Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa\") failed: Property not found. retry in 1.453292592s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:03.222194301Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a\") failed: Property not found. retry in 1.347344347s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:03.230045097Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b\") failed: Property not found. retry in 659.865857ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:03.420352597Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465\") failed: Property not found. retry in 529.799556ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:03.468329256Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9\") failed: Property not found. retry in 838.589487ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:03.537856647Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7\") failed: Property not found. retry in 893.170702ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:03.568819152Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c\") failed: Property not found. retry in 701.028738ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:03.576471503Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc\") failed: Property not found. retry in 1.390167238s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:03.584221897Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3\") failed: Property not found. retry in 963.947665ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:03.607799547Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862\") failed: Property not found. retry in 627.344597ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:03.749663846Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095\") failed: Property not found. retry in 1.027407224s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:03.757224026Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046\") failed: Property not found. retry in 1.367442835s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:03.834704295Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d\") failed: Property not found. retry in 625.428331ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:03.8787Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480\") failed: Property not found. retry in 805.40977ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:03.956110158Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465\") failed: Property not found. retry in 587.73506ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:04.007353325Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876\") failed: Property not found. retry in 825.721099ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:04.080544233Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b\") failed: Property not found. retry in 1.152680622s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:04.09135166Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e\") failed: Property not found. retry in 1.005571705s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:04.121773412Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79\") failed: Property not found. retry in 1.481577477s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:04.164911887Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2\") failed: Property not found. retry in 640.06715ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:04.179623206Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c\") failed: Property not found. retry in 1.408361753s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:04.192770808Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c\") failed: Property not found. retry in 583.747966ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:04.289755281Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9\") failed: Property not found. retry in 588.546793ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:04.387167855Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862\") failed: Property not found. retry in 1.476708464s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:04.395622306Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7\") failed: Property not found. retry in 786.984016ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:04.471149489Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d\") failed: Property not found. retry in 582.628446ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:04.516048545Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3\") failed: Property not found. retry in 1.443464523s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:04.58489715Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a\") failed: Property not found. retry in 1.185153166s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:04.598046242Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465\") failed: Property not found. retry in 1.244441889s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:04.608814788Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa\") failed: Property not found. retry in 920.542968ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:04.690809505Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480\") failed: Property not found. retry in 547.071478ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:04.774245466Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095\") failed: Property not found. retry in 1.43137454s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:04.796218538Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2\") failed: Property not found. retry in 598.410726ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:04.831790853Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c\") failed: Property not found. retry in 654.9178ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:04.886246556Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876\") failed: Property not found. retry in 1.384886378s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:04.92180438Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc\") failed: Property not found. retry in 1.391208911s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:04.99717493Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9\") failed: Property not found. retry in 613.393346ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:05.013792445Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d\") failed: Property not found. retry in 1.346065907s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:05.11550789Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e\") failed: Property not found. retry in 953.877434ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:05.144175661Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046\") failed: Property not found. retry in 512.365712ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:05.221581009Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b\") failed: Property not found. retry in 852.959817ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:05.262352798Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7\") failed: Property not found. retry in 594.378212ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:05.278622614Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480\") failed: Property not found. retry in 1.238066207s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:05.342141103Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2\") failed: Property not found. retry in 1.315565091s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:05.445319593Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c\") failed: Property not found. retry in 629.877845ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:05.559245548Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c\") failed: Property not found. retry in 970.972461ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:05.616907537Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa\") failed: Property not found. retry in 1.155659569s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:05.630745366Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9\") failed: Property not found. retry in 794.259435ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:05.644311238Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79\") failed: Property not found. retry in 540.503761ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:05.66166704Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046\") failed: Property not found. retry in 1.295600662s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:05.750535641Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a\") failed: Property not found. retry in 1.006393582s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:05.798946251Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465\") failed: Property not found. retry in 756.727781ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:05.869427784Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7\") failed: Property not found. retry in 1.011246548s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:05.886014008Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862\") failed: Property not found. retry in 1.030796806s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:05.90039485Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3\") failed: Property not found. retry in 661.425914ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:06.045735892Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e\") failed: Property not found. retry in 1.003564486s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:06.115615741Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b\") failed: Property not found. retry in 528.806356ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:06.131335294Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c\") failed: Property not found. retry in 778.027565ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:06.203821086Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79\") failed: Property not found. retry in 888.080612ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:06.217585273Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095\") failed: Property not found. retry in 1.258616118s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:06.24485803Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876\") failed: Property not found. retry in 1.205911159s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:06.318843828Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc\") failed: Property not found. retry in 1.257067845s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:06.400905327Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9\") failed: Property not found. retry in 1.29192886s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:06.451474277Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d\") failed: Property not found. retry in 1.328237469s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:06.474355491Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c\") failed: Property not found. retry in 839.872034ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:06.607316708Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465\") failed: Property not found. retry in 928.024179ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:06.637570596Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b\") failed: Property not found. retry in 1.143472534s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:06.672158018Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3\") failed: Property not found. retry in 1.43169962s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:06.719199145Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2\") failed: Property not found. retry in 566.220388ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:06.771825305Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa\") failed: Property not found. retry in 1.138022568s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:06.797195017Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a\") failed: Property not found. retry in 1.27333666s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:06.815335148Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7\") failed: Property not found. retry in 1.335819115s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:06.907452546Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862\") failed: Property not found. retry in 1.458677163s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:06.941971086Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c\") failed: Property not found. retry in 714.797094ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:06.99099318Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046\") failed: Property not found. retry in 1.475182936s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:07.055939202Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79\") failed: Property not found. retry in 1.122916619s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:07.114842381Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e\") failed: Property not found. retry in 1.408205128s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:07.129413157Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095\") failed: Property not found. retry in 1.398048992s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:07.605296069Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c\") failed: Property not found. retry in 865.910902ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:07.613975515Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876\") failed: Property not found. retry in 1.434145982s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:07.622242421Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465\") failed: Property not found. retry in 544.577551ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:07.630784254Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc\") failed: Property not found. retry in 1.297798619s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:07.666914802Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c\") failed: Property not found. retry in 1.003862399s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:07.713804906Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9\") failed: Property not found. retry in 750.815675ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:07.751666065Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d\") failed: Property not found. retry in 1.186019981s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:07.81085092Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b\") failed: Property not found. retry in 797.104249ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:07.825239152Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa\") failed: Property not found. retry in 673.62495ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:07.944791571Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a\") failed: Property not found. retry in 1.164813402s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:08.098030291Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3\") failed: Property not found. retry in 540.098897ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:08.160888484Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465\") failed: Property not found. retry in 1.036108339s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:08.184489664Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79\") failed: Property not found. retry in 953.191883ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:08.246906026Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7\") failed: Property not found. retry in 1.346235289s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:08.255351557Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862\") failed: Property not found. retry in 820.340184ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:08.411198298Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c\") failed: Property not found. retry in 1.158796691s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:08.490300667Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9\") failed: Property not found. retry in 1.4604962s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:08.51191134Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046\") failed: Property not found. retry in 1.095409857s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:08.540768375Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e\") failed: Property not found. retry in 725.853315ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:08.549692986Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa\") failed: Property not found. retry in 1.468642945s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:08.627387191Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b\") failed: Property not found. retry in 530.000998ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:08.635407842Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3\") failed: Property not found. retry in 1.339465622s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:08.711591511Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c\") failed: Property not found. retry in 1.42021812s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:08.726343802Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc\") failed: Property not found. retry in 708.308815ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:08.976663717Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d\") failed: Property not found. retry in 1.322168562s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:09.008026491Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095\") failed: Property not found. retry in 833.191167ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:09.01553343Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876\") failed: Property not found. retry in 1.475454757s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:09.066451989Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b\") failed: Property not found. retry in 518.39515ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:09.176726848Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79\") failed: Property not found. retry in 1.373480999s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:09.209753582Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465\") failed: Property not found. retry in 1.468661622s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:09.22985991Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862\") failed: Property not found. retry in 1.435637814s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:09.240413381Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a\") failed: Property not found. retry in 1.395025414s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:09.273351213Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e\") failed: Property not found. retry in 748.933897ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:09.284999349Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7\") failed: Property not found. retry in 1.23586947s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:09.614830972Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046\") failed: Property not found. retry in 1.311832728s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:09.646143226Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c\") failed: Property not found. retry in 1.287324151s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:09.658495809Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc\") failed: Property not found. retry in 819.573776ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:09.695245272Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b\") failed: Property not found. retry in 1.15383376s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:09.703678912Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095\") failed: Property not found. retry in 1.424088924s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:09.859259027Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9\") failed: Property not found. retry in 661.418982ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:09.982249598Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e\") failed: Property not found. retry in 513.444686ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:10.044078596Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3\") failed: Property not found. retry in 574.842712ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:10.061360817Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa\") failed: Property not found. retry in 1.233645214s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:10.106264733Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c\") failed: Property not found. retry in 1.460191433s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:10.156107057Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:10.185566727Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","component":"MetricsManager","sled_id":"8fa4bc78-1026-41f0-b09d-c67224071376","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"92.760311014s"} {"msg":"wait for service Some(\"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d\") failed: Property not found. retry in 802.070966ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:10.34026841Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc\") failed: Property not found. retry in 741.628336ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:10.526284128Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876\") failed: Property not found. retry in 851.590942ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:10.552327536Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79\") failed: Property not found. retry in 691.892413ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:10.592569342Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a\") failed: Property not found. retry in 944.940253ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:10.676270849Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862\") failed: Property not found. retry in 909.886088ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:10.684066434Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465\") failed: Property not found. retry in 1.239706361s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:10.706469867Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862\") failed: Property not found. retry in 656.074483ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:11.612400031Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862\") failed: Property not found. retry in 591.370629ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:12.288886717Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862\") failed: Property not found. retry in 1.068722819s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:12.903995115Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862\") failed: Property not found. retry in 934.474786ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:13.993469197Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/8fa4bc78-1026-41f0-b09d-c67224071376): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-07-25T05:48:14.638527315Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"8fa4bc78-1026-41f0-b09d-c67224071376","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/8fa4bc78-1026-41f0-b09d-c67224071376): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:14.63874959Z","hostname":"buskin","pid":1555,"component":"NexusNotifierTask","sled_id":"8fa4bc78-1026-41f0-b09d-c67224071376","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"wait for service Some(\"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862\") failed: Property not found. retry in 799.660427ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:14.942044713Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/8fa4bc78-1026-41f0-b09d-c67224071376): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:15.15919285Z","hostname":"buskin","pid":1555,"component":"NexusNotifierTask","sled_id":"8fa4bc78-1026-41f0-b09d-c67224071376","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/8fa4bc78-1026-41f0-b09d-c67224071376): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-07-25T05:48:15.642038475Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"8fa4bc78-1026-41f0-b09d-c67224071376","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:15.772790321Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_crucible_655516f5-b8a5-4e1d-b3e9-1b4fd5e5717c"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:15.772874933Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_external_dns_2c3f8453-233f-428f-93cc-0bfefb84929d"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:15.772884333Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_crucible_1f722dd3-3360-4e20-99bf-1de5eb86ffd9"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:15.772890823Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_nexus_85472347-dfd9-425e-bd98-9b81ebdf3b2b"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:15.772897073Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_crucible_d94d9602-d610-44ce-b9fc-06ded41a1480"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:15.772915914Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_crucible_a0b9fba0-c330-402f-9bc0-a7510786c98a"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:15.772922264Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_crucible_7f7f1d81-12f9-4239-8e97-9582c9218b79"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:15.772926794Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_nexus_e4efbc31-e513-4290-82f5-c51d8afea0c7"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:15.772931194Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_crucible_dfb1f626-0f55-4385-87de-de59052d98d2"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:15.772935544Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_external_dns_b04db43b-dc24-43ce-9d97-32193f88e046"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:15.772940074Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_crucible_adff3617-5320-4e42-8b10-1430a1eb0afc"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:15.772947274Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_crucible_pantry_a635cc62-7516-4fd9-b490-62d6de386095"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:15.772951824Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_crucible_ee1bae54-08c6-4724-bc97-b5188feb1b8c"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:15.772956275Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_nexus_221c5e7c-0832-4890-8b0a-f9ba08725876"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:15.772972205Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_clickhouse_4ba7467e-cf15-4665-9aba-eeedcaaaa862"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:15.772977045Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_oximeter_4b4998f4-10de-49f2-9326-b482137c5fd3"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:15.772981685Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_crucible_pantry_ac9fa851-e7e1-4295-ac79-d6076526f465"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:15.77320133Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_crucible_pantry_1cb96976-2047-44b8-a79c-9fcd74d17bfa"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:15.773209381Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3062","zone":"oxz_crucible_2e3ea66c-7cbd-4cae-aca1-5ebcb142282e"} {"msg":"Writing ledger to /pool/int/6d2aa15e-618e-402e-b368-cf27fd8def68/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:15.774255495Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/75c788e1-45ef-4fcf-958d-db07331bef52/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:15.77780578Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:15.781343074Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"919d718d-e5d0-47cf-8e40-ace8413870d3","remote_addr":"[fd00:1122:3344:101::1]:47495","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":39919954,"response_code":"204"} {"msg":"Finished setting up services","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:15.793272457Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1247"} {"msg":"Writing ledger to /pool/int/6d2aa15e-618e-402e-b368-cf27fd8def68/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:15.793436581Z","hostname":"buskin","pid":1555,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/75c788e1-45ef-4fcf-958d-db07331bef52/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:15.793735228Z","hostname":"buskin","pid":1555,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Handing off control to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:15.796066723Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:684"} {"msg":"Nexus address: [fd00:1122:3344:101::c]:12221","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:15.798838079Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:697"} {"msg":"rack_network_config: RackNetworkConfigV2 {\n bfd: [],\n bgp: [],\n infra_ip_first: 10.151.1.174,\n infra_ip_last: 10.151.1.174,\n ports: [\n PortConfigV2 {\n addresses: [\n UplinkAddressConfig {\n address: V4(\n Ipv4Net {\n addr: 10.151.1.174,\n width: 24,\n },\n ),\n vlan_id: None,\n },\n ],\n autoneg: false,\n bgp_peers: [],\n port: \"qsfp0\",\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n nexthop: 10.151.1.100,\n vlan_id: None,\n },\n ],\n switch: Switch0,\n uplink_port_fec: None,\n uplink_port_speed: Speed40G,\n },\n ],\n rack_subnet: Ipv6Net {\n addr: fd00:1122:3344:100::,\n width: 56,\n },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:15.82960929Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:828"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/b4c6410d-62e1-4369-a8c0-f8e6399dd808/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:15.831438313Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:890"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/b4c6410d-62e1-4369-a8c0-f8e6399dd808/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:15.941463186Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:890"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/b4c6410d-62e1-4369-a8c0-f8e6399dd808/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:16.047784421Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:890"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/b4c6410d-62e1-4369-a8c0-f8e6399dd808/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:16.130019694Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:890"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/b4c6410d-62e1-4369-a8c0-f8e6399dd808/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:16.318259534Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:890"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/b4c6410d-62e1-4369-a8c0-f8e6399dd808/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:16.610615077Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:890"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/8fa4bc78-1026-41f0-b09d-c67224071376): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-07-25T05:48:16.64484624Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"8fa4bc78-1026-41f0-b09d-c67224071376","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/b4c6410d-62e1-4369-a8c0-f8e6399dd808/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:16.904206669Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:890"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/8fa4bc78-1026-41f0-b09d-c67224071376): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:17.156246484Z","hostname":"buskin","pid":1555,"component":"NexusNotifierTask","sled_id":"8fa4bc78-1026-41f0-b09d-c67224071376","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/b4c6410d-62e1-4369-a8c0-f8e6399dd808/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:17.221884683Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:890"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/b4c6410d-62e1-4369-a8c0-f8e6399dd808/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:17.580321664Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:890"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/8fa4bc78-1026-41f0-b09d-c67224071376): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-07-25T05:48:17.647363207Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"8fa4bc78-1026-41f0-b09d-c67224071376","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/b4c6410d-62e1-4369-a8c0-f8e6399dd808/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:17.888694588Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:890"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/b4c6410d-62e1-4369-a8c0-f8e6399dd808/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:18.199083489Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:890"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/b4c6410d-62e1-4369-a8c0-f8e6399dd808/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:18.551387125Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:890"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/8fa4bc78-1026-41f0-b09d-c67224071376): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-07-25T05:48:18.65098873Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"8fa4bc78-1026-41f0-b09d-c67224071376","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/b4c6410d-62e1-4369-a8c0-f8e6399dd808/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:19.062585605Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:890"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/8fa4bc78-1026-41f0-b09d-c67224071376): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:19.157107869Z","hostname":"buskin","pid":1555,"component":"NexusNotifierTask","sled_id":"8fa4bc78-1026-41f0-b09d-c67224071376","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/8fa4bc78-1026-41f0-b09d-c67224071376): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-07-25T05:48:19.654550252Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"8fa4bc78-1026-41f0-b09d-c67224071376","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/b4c6410d-62e1-4369-a8c0-f8e6399dd808/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:20.215082413Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:890"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/8fa4bc78-1026-41f0-b09d-c67224071376): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-07-25T05:48:20.657516759Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"8fa4bc78-1026-41f0-b09d-c67224071376","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/8fa4bc78-1026-41f0-b09d-c67224071376): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:21.156687364Z","hostname":"buskin","pid":1555,"component":"NexusNotifierTask","sled_id":"8fa4bc78-1026-41f0-b09d-c67224071376","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/b4c6410d-62e1-4369-a8c0-f8e6399dd808/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:21.337128479Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:890"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/8fa4bc78-1026-41f0-b09d-c67224071376): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-07-25T05:48:21.659946925Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"8fa4bc78-1026-41f0-b09d-c67224071376","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/b4c6410d-62e1-4369-a8c0-f8e6399dd808/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:22.583480796Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:890"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/8fa4bc78-1026-41f0-b09d-c67224071376): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-07-25T05:48:22.66319661Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"8fa4bc78-1026-41f0-b09d-c67224071376","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/8fa4bc78-1026-41f0-b09d-c67224071376): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:48:23.156386252Z","hostname":"buskin","pid":1555,"component":"NexusNotifierTask","sled_id":"8fa4bc78-1026-41f0-b09d-c67224071376","component":"SledAgent","file":"sled-agent/src/nexus.rs:619"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/8fa4bc78-1026-41f0-b09d-c67224071376): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-07-25T05:48:23.665751588Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"8fa4bc78-1026-41f0-b09d-c67224071376","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"598fdfcb-1e35-4345-9c99-c06ae105979b\", \"content-length\": \"173\", \"date\": \"Thu, 25 Jul 2024 05:48:24 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: rack with id \\\"b4c6410d-62e1-4369-a8c0-f8e6399dd808\\\"\", request_id: \"598fdfcb-1e35-4345-9c99-c06ae105979b\" }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:24.135139785Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:890"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:24.948455369Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:39843"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:24.948737326Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"ee157287-dfd9-4f15-a1c9-c673da533b35","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:39843","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":59,"response_code":"200"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"950bc629-e843-4819-95da-4705da84e502\", \"content-length\": \"173\", \"date\": \"Thu, 25 Jul 2024 05:48:26 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"8fa4bc78-1026-41f0-b09d-c67224071376\\\"\", request_id: \"950bc629-e843-4819-95da-4705da84e502\" }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:26.532057297Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:890"} {"msg":"Successfully put SledAgentInfo to nexus","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:27.194489557Z","hostname":"buskin","pid":1555,"component":"NexusNotifierTask","sled_id":"8fa4bc78-1026-41f0-b09d-c67224071376","component":"SledAgent","file":"sled-agent/src/nexus.rs:592","new_generation":"1","old_generation":"1"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:27.32641397Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:61068"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:27.327807954Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:27.415390164Z","hostname":"buskin","pid":1555,"uri":"/inventory","method":"GET","req_id":"d672f128-99e9-4e36-beae-65884ed4d4f9","remote_addr":"[fd00:1122:3344:101::c]:61068","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":88886,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:27.416459299Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/6d2aa15e-618e-402e-b368-cf27fd8def68/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:27.416535361Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/75c788e1-45ef-4fcf-958d-db07331bef52/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:27.417888193Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:27.418273012Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"GET","req_id":"463d6462-bdb6-4647-b7a6-310b06d1b2d8","remote_addr":"[fd00:1122:3344:101::c]:61068","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":2562,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:28.6355915Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:44913"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:28.638423747Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:673","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: a6846ef5-8686-4c7e-ae28-d178a4029a85, kind: Service { id: 2c3f8453-233f-428f-93cc-0bfefb84929d }, name: Name(\"external-dns-2c3f8453-233f-428f-93cc-0bfefb84929d\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 164, 190])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: f04fae05-ec27-43ac-ad4d-ab63c091bf1b, kind: Service { id: b04db43b-dc24-43ce-9d97-32193f88e046 }, name: Name(\"external-dns-b04db43b-dc24-43ce-9d97-32193f88e046\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 154])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 735e2401-2ba5-4079-b4a0-7f0184716575, kind: Service { id: e4efbc31-e513-4290-82f5-c51d8afea0c7 }, name: Name(\"nexus-e4efbc31-e513-4290-82f5-c51d8afea0c7\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 154])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 9e8859b7-3d6e-4066-b58c-24fc535c1122, kind: Service { id: 221c5e7c-0832-4890-8b0a-f9ba08725876 }, name: Name(\"nexus-221c5e7c-0832-4890-8b0a-f9ba08725876\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 155])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: f33554f0-0c2d-4b66-98d0-73365d475c4b, kind: Service { id: 85472347-dfd9-425e-bd98-9b81ebdf3b2b }, name: Name(\"nexus-85472347-dfd9-425e-bd98-9b81ebdf3b2b\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 244])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:28.638461878Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:28.638501629Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:28.638513079Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:28.63853054Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:28.63856275Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:28.638596001Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:28.638617192Z","hostname":"buskin","pid":1555,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"2c2fb3cd-7dcd-4d96-921b-6b0af06f799d","remote_addr":"[fd00:1122:3344:101::c]:44913","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":2954,"response_code":"204"} {"msg":"Handoff to Nexus is complete","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:28.641201093Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:900"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:28.892276446Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:45601"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:28.892424299Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:28.969346796Z","hostname":"buskin","pid":1555,"uri":"/inventory","method":"GET","req_id":"3edc927e-750e-419d-b536-465d019f7bff","remote_addr":"[fd00:1122:3344:101::c]:45601","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":77020,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:28.969611262Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Reading ledger from /pool/int/6d2aa15e-618e-402e-b368-cf27fd8def68/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:28.969677914Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/75c788e1-45ef-4fcf-958d-db07331bef52/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:28.970189436Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:28.971129848Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"GET","req_id":"3032e27f-7003-47f1-85b5-d5fd8a005c60","remote_addr":"[fd00:1122:3344:101::c]:45601","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":1594,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:29.900950278Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:53469"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:29.901193034Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:673","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: a6846ef5-8686-4c7e-ae28-d178a4029a85, kind: Service { id: 2c3f8453-233f-428f-93cc-0bfefb84929d }, name: Name(\"external-dns-2c3f8453-233f-428f-93cc-0bfefb84929d\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 164, 190])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: f04fae05-ec27-43ac-ad4d-ab63c091bf1b, kind: Service { id: b04db43b-dc24-43ce-9d97-32193f88e046 }, name: Name(\"external-dns-b04db43b-dc24-43ce-9d97-32193f88e046\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 154])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 735e2401-2ba5-4079-b4a0-7f0184716575, kind: Service { id: e4efbc31-e513-4290-82f5-c51d8afea0c7 }, name: Name(\"nexus-e4efbc31-e513-4290-82f5-c51d8afea0c7\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 154])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 9e8859b7-3d6e-4066-b58c-24fc535c1122, kind: Service { id: 221c5e7c-0832-4890-8b0a-f9ba08725876 }, name: Name(\"nexus-221c5e7c-0832-4890-8b0a-f9ba08725876\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 155])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: f33554f0-0c2d-4b66-98d0-73365d475c4b, kind: Service { id: 85472347-dfd9-425e-bd98-9b81ebdf3b2b }, name: Name(\"nexus-85472347-dfd9-425e-bd98-9b81ebdf3b2b\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 244])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:29.901237275Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:29.901274116Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:29.901293986Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:29.901310516Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:29.901324717Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:29.901358928Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:29.901374738Z","hostname":"buskin","pid":1555,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"36f6d103-f86a-4733-ab6f-55b295a88e52","remote_addr":"[fd00:1122:3344:101::a]:53469","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":379,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:30.023953379Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:47040"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:30.024196185Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:673","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: a6846ef5-8686-4c7e-ae28-d178a4029a85, kind: Service { id: 2c3f8453-233f-428f-93cc-0bfefb84929d }, name: Name(\"external-dns-2c3f8453-233f-428f-93cc-0bfefb84929d\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 164, 190])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: f04fae05-ec27-43ac-ad4d-ab63c091bf1b, kind: Service { id: b04db43b-dc24-43ce-9d97-32193f88e046 }, name: Name(\"external-dns-b04db43b-dc24-43ce-9d97-32193f88e046\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 154])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 735e2401-2ba5-4079-b4a0-7f0184716575, kind: Service { id: e4efbc31-e513-4290-82f5-c51d8afea0c7 }, name: Name(\"nexus-e4efbc31-e513-4290-82f5-c51d8afea0c7\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 154])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 9e8859b7-3d6e-4066-b58c-24fc535c1122, kind: Service { id: 221c5e7c-0832-4890-8b0a-f9ba08725876 }, name: Name(\"nexus-221c5e7c-0832-4890-8b0a-f9ba08725876\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 155])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: f33554f0-0c2d-4b66-98d0-73365d475c4b, kind: Service { id: 85472347-dfd9-425e-bd98-9b81ebdf3b2b }, name: Name(\"nexus-85472347-dfd9-425e-bd98-9b81ebdf3b2b\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 244])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:30.024259586Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:30.024302467Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:30.024332648Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:30.024357409Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:30.024377929Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:30.024590054Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:30.024626155Z","hostname":"buskin","pid":1555,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"8bdcc0f4-a25b-4d91-8d7f-34c76f144340","remote_addr":"[fd00:1122:3344:101::c]:47040","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":603,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:30.173989872Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:36519"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:30.174813471Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:673","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: a6846ef5-8686-4c7e-ae28-d178a4029a85, kind: Service { id: 2c3f8453-233f-428f-93cc-0bfefb84929d }, name: Name(\"external-dns-2c3f8453-233f-428f-93cc-0bfefb84929d\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 164, 190])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: f04fae05-ec27-43ac-ad4d-ab63c091bf1b, kind: Service { id: b04db43b-dc24-43ce-9d97-32193f88e046 }, name: Name(\"external-dns-b04db43b-dc24-43ce-9d97-32193f88e046\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 154])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 735e2401-2ba5-4079-b4a0-7f0184716575, kind: Service { id: e4efbc31-e513-4290-82f5-c51d8afea0c7 }, name: Name(\"nexus-e4efbc31-e513-4290-82f5-c51d8afea0c7\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 154])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 9e8859b7-3d6e-4066-b58c-24fc535c1122, kind: Service { id: 221c5e7c-0832-4890-8b0a-f9ba08725876 }, name: Name(\"nexus-221c5e7c-0832-4890-8b0a-f9ba08725876\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 155])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: f33554f0-0c2d-4b66-98d0-73365d475c4b, kind: Service { id: 85472347-dfd9-425e-bd98-9b81ebdf3b2b }, name: Name(\"nexus-85472347-dfd9-425e-bd98-9b81ebdf3b2b\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 244])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:30.174846052Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:30.174884383Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:30.174895833Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:30.174902064Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:30.174918054Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:30.174935504Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:30.174951925Z","hostname":"buskin","pid":1555,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"8b8a2c18-d462-44a4-8948-9cbe7eb1d6e7","remote_addr":"[fd00:1122:3344:101::b]:36519","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":841,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:37.165556605Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:50208"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:37.165850132Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"02ac38d5-fb24-4524-9323-5821edc6406a","remote_addr":"[fd00:1122:3344:101::c]:50208","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":261,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:37.23020679Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"36629143-5151-43c5-bac7-427f5368d67d","remote_addr":"[fd00:1122:3344:101::c]:50208","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":2745,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:54.473911849Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:39689"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:54.474053002Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"4d6aad7c-3de0-4cea-95c6-490806bbd07a","remote_addr":"[fd00:1122:3344:101::a]:39689","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":110,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:54.481284784Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"f5d28281-d231-40e4-8800-5924bfb4b4b1","remote_addr":"[fd00:1122:3344:101::a]:39689","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":118,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:54.518793224Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:36372"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:54.51987943Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"ed729cb7-7d1d-4624-bc8e-e07b5e375d82","remote_addr":"[fd00:1122:3344:101::a]:36372","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":1065,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:54.661749668Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:58226"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:54.662126387Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3824"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:54.714233445Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3835"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:54.746325317Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"08d86f4f-8774-4655-a70a-ddfdeffa2239","remote_addr":"[fd00:1122:3344:101::a]:58226","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":84515,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:54.747438873Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"e6d37640-30f2-414f-a6e9-8a4feca889cf","remote_addr":"[fd00:1122:3344:101::a]:58226","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":821,"response_code":"200"} {"msg":"Attempting to update network config with generation: 2, current_generation: 1","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:54.78100292Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/6d2aa15e-618e-402e-b368-cf27fd8def68/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:54.781049131Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/75c788e1-45ef-4fcf-958d-db07331bef52/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:54.781979464Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 2","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:54.78311826Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:54.783150251Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"PUT","req_id":"c3ed24c5-062b-418c-99ee-35eb61574722","remote_addr":"[fd00:1122:3344:101::a]:58226","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":3511,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:54.853777178Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:56316"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:54.853867871Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"3729db6f-250e-4eb4-8a07-7ff4f7ba91e5","remote_addr":"[fd00:1122:3344:101::c]:56316","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":67,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:54.861507062Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"c1135637-21a8-46f5-9137-d8b1352c42cc","remote_addr":"[fd00:1122:3344:101::c]:56316","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":152,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:54.982947056Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:44425"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:54.982994397Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"f0a49be1-aaf3-4e32-a545-fe1708e428e6","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:44425","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":29,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:55.002481049Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:34075"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:55.002787377Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"fc0df957-46da-4029-8f6e-b57e55bff1af","remote_addr":"[fd00:1122:3344:101::c]:34075","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":133,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:55.048601195Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:63210"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:55.048699777Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3824"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:55.074619883Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3835"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:55.095380556Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"45af84a3-97ed-47d6-9094-684d981136c8","remote_addr":"[fd00:1122:3344:101::c]:63210","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":46735,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:55.095681173Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"a684c1d9-64b9-4402-b436-ac8b7f01e5df","remote_addr":"[fd00:1122:3344:101::c]:63210","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":98,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:55.15830936Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:54069"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:55.159111839Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"e505292e-64f8-4b80-83d5-30ebea24326e","remote_addr":"[fd00:1122:3344:101::b]:54069","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":729,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:55.167191641Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"26103702-4dda-44a0-b21f-9e946cb72f6d","remote_addr":"[fd00:1122:3344:101::b]:54069","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":111,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:55.204977878Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:55742"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:55.20587205Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"5b2f9e33-db37-4efc-9950-0f8933d26fb3","remote_addr":"[fd00:1122:3344:101::b]:55742","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":837,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:55.228136868Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:54636"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:55.22822689Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3824"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:55.251759629Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3835"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:55.271623441Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"23e1497a-69cb-4eea-aaa4-0d0f19c1e053","remote_addr":"[fd00:1122:3344:101::b]:54636","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":43425,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:48:55.271917138Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"b8b655d4-301a-4311-8950-4de1a0ff1f42","remote_addr":"[fd00:1122:3344:101::b]:54636","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":80,"response_code":"200"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:49:15.692631856Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:49:24.481391384Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:48397"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:49:24.482238264Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"c4df246a-64ce-4c1f-9a83-d3446c3d230e","remote_addr":"[fd00:1122:3344:101::a]:48397","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":817,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:49:24.489743332Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"0b5a3b1c-6676-47c1-b71a-fee42786a78d","remote_addr":"[fd00:1122:3344:101::a]:48397","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":123,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:49:24.566964396Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:47540"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:49:24.5671345Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"5efcdc7d-727e-4979-ae10-ad9120dc228a","remote_addr":"[fd00:1122:3344:101::a]:47540","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":114,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:49:24.897484484Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:51391"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:49:24.897589147Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3824"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:49:24.921584877Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3835"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:49:24.944976162Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"42582f48-1d56-48fa-b6a3-529664e84c77","remote_addr":"[fd00:1122:3344:101::a]:51391","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":47437,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:49:24.945573306Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"e065b31d-3665-44e1-a663-ffbacbbfe7de","remote_addr":"[fd00:1122:3344:101::a]:51391","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":210,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:49:25.015507177Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:57676"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:49:25.015574489Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"4788cb1a-8e1b-4e62-bce4-c01a65361476","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:57676","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":43,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:49:25.119548878Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:48565"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:49:25.119627309Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"8d9b7045-35e2-4fe0-b1aa-a59d68aa30a5","remote_addr":"[fd00:1122:3344:101::b]:48565","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":38,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:49:25.127421294Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"b6c1c553-19fb-4923-bc09-1559d1329429","remote_addr":"[fd00:1122:3344:101::b]:48565","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":104,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:49:25.163604694Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:46355"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:49:25.164661839Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"e0f5301e-a742-44c0-8d92-96a61a3be012","remote_addr":"[fd00:1122:3344:101::c]:46355","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":1007,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:49:25.171736567Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"319aa3ba-d8ec-419c-85ec-6c9ac7bd8eb7","remote_addr":"[fd00:1122:3344:101::c]:46355","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":70,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:49:25.196354541Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:37458"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:49:25.196695119Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"6d6392ba-9e4e-47c8-9bf9-1cd5c851dcb1","remote_addr":"[fd00:1122:3344:101::b]:37458","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":284,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:49:25.219416609Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:33594"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:49:25.219540222Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"fa2d985d-42d1-45a1-a219-619eaa69474e","remote_addr":"[fd00:1122:3344:101::c]:33594","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":97,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:49:25.342937242Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:57997"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:49:25.343111716Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3824"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:49:25.370782893Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3835"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:49:25.395016369Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"335b2984-a633-4e87-a404-9d7427e36903","remote_addr":"[fd00:1122:3344:101::c]:57997","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":52040,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:49:25.396159436Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"c1b50b4e-939c-4599-9a59-9f43baea893f","remote_addr":"[fd00:1122:3344:101::c]:57997","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":916,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:49:25.494318077Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:64881"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:49:25.49445967Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3824"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:49:25.520290334Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3835"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:49:25.545175634Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"58e7261b-a7a6-4f39-92e3-aa247181e3d3","remote_addr":"[fd00:1122:3344:101::b]:64881","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":50774,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:49:25.545467971Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"81ec800f-47aa-4337-b629-2b1fa007fe71","remote_addr":"[fd00:1122:3344:101::b]:64881","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":84,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:49:54.350680948Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:56765"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:49:54.350812282Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"f0caddde-402a-4c04-808c-5e18a4b1501f","remote_addr":"[fd00:1122:3344:101::a]:56765","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":117,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:49:54.358035103Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"c8982c67-b69e-43a8-b27f-2a1f49e408b1","remote_addr":"[fd00:1122:3344:101::a]:56765","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":793,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:49:54.43961632Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:59998"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:49:54.439781234Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"48d8ab32-807e-4923-9867-37bbf633007e","remote_addr":"[fd00:1122:3344:101::a]:59998","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":144,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:49:54.538861397Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:42335"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:49:54.538948519Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3824"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:49:54.56638185Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3835"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:49:54.588860474Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"6c2931f9-58d7-40b0-819b-82ae9c787074","remote_addr":"[fd00:1122:3344:101::a]:42335","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":49948,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:49:54.589252833Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"c3a890c2-a576-4ed0-8c1b-9fa9664d197e","remote_addr":"[fd00:1122:3344:101::a]:42335","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":151,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:49:54.868635197Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:45679"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:49:54.86876106Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"1d2c0793-584e-4f29-a055-4654f27cb8ab","remote_addr":"[fd00:1122:3344:101::c]:45679","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":108,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:49:54.875276225Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"18d4c648-4ef2-4ed0-9c22-4c9332c0cc6a","remote_addr":"[fd00:1122:3344:101::c]:45679","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":118,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:49:54.911623268Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:63074"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:49:54.911763511Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"088a65aa-8cd6-4aaf-bd69-987c11b199f1","remote_addr":"[fd00:1122:3344:101::c]:63074","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":121,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:49:54.911907495Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:63208"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:49:54.912006567Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"752bc221-148d-4c03-9903-4d06ff231673","remote_addr":"[fd00:1122:3344:101::b]:63208","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":34,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:49:54.918566023Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"84bf5c38-7e67-40c5-a3cc-fed7ec44c91e","remote_addr":"[fd00:1122:3344:101::b]:63208","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":58,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:49:55.020213117Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:49266"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:49:55.020381001Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"5af92990-d7e5-480c-bfa7-2d6def609025","remote_addr":"[fd00:1122:3344:101::b]:49266","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":132,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:49:55.038766898Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:35944"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:49:55.038837309Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3824"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:49:55.062425539Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3835"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:49:55.083696184Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"2eb03efa-20da-489d-86fa-ef4bc9f5846e","remote_addr":"[fd00:1122:3344:101::c]:35944","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":44890,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:49:55.083834938Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:34433"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:49:55.083965261Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"76edc706-bc72-4a60-a454-9554e188e7f6","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:34433","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":61,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:49:55.084099274Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"48d9b7dc-0502-4716-93ee-594aac110b1f","remote_addr":"[fd00:1122:3344:101::c]:35944","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":154,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:49:55.113991834Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:52711"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:49:55.114116277Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3824"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:49:55.135612377Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3835"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:49:55.156588975Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"15174613-52d7-40f3-993c-d1a2a83c9442","remote_addr":"[fd00:1122:3344:101::b]:52711","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":42514,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:49:55.156834491Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"4aa88312-3d2c-49c9-aff9-cd9bc55cbe61","remote_addr":"[fd00:1122:3344:101::b]:52711","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":85,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:50:13.055627265Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:57429","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","component":"producer-server","component":"MetricsManager","sled_id":"8fa4bc78-1026-41f0-b09d-c67224071376","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::d]:46986"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:50:13.064810873Z","hostname":"buskin","pid":1555,"uri":"/8fa4bc78-1026-41f0-b09d-c67224071376","method":"GET","req_id":"0dbcaf0c-c379-4655-b3f0-c06be8b85a38","remote_addr":"[fd00:1122:3344:101::d]:46986","local_addr":"[fd00:1122:3344:101::1]:57429","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","component":"producer-server","component":"MetricsManager","sled_id":"8fa4bc78-1026-41f0-b09d-c67224071376","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":8213,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:50:24.694201834Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:55261"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:50:24.694347498Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"c3f168c5-5b4d-4e2f-9f46-8dd78b0f1cb9","remote_addr":"[fd00:1122:3344:101::a]:55261","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":102,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:50:24.701731023Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"edf4b9ea-4a2c-477e-afaf-e5ec46de4965","remote_addr":"[fd00:1122:3344:101::a]:55261","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":133,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:50:24.760955949Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:61931"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:50:24.762895576Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"0df04db0-ce03-4261-9cc2-e5132603247e","remote_addr":"[fd00:1122:3344:101::a]:61931","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":1896,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:50:24.881593554Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:61883"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:50:24.881710017Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3824"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:50:24.911951765Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3835"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:50:24.940178125Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"8a259190-82fd-42c5-85a7-8e0dfbc0514a","remote_addr":"[fd00:1122:3344:101::a]:61883","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":58525,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:50:24.940438072Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"e5769390-6a41-4643-aac9-ea8e29ccdfca","remote_addr":"[fd00:1122:3344:101::a]:61883","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":99,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:50:25.020640346Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:47161"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:50:25.020767249Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"93e3afa7-07d0-4b7b-be13-4d213807b305","remote_addr":"[fd00:1122:3344:101::b]:47161","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":116,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:50:25.063817901Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:58235"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:50:25.063924464Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"d4b6eb3e-25de-4468-8d88-61f89329aa25","remote_addr":"[fd00:1122:3344:101::c]:58235","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":80,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:50:25.065470841Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:36964"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:50:25.066399233Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"33f10fbf-19c7-4232-81c5-11e076454d1f","remote_addr":"[fd00:1122:3344:101::c]:36964","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":905,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:50:25.074201008Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"45c90186-277a-4f2e-a262-4e345f2a7b77","remote_addr":"[fd00:1122:3344:101::c]:36964","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":95,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:50:25.117977478Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:60989"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:50:25.118035679Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"cedb7a5d-574c-4c7d-a28c-dd9c6880dfa2","remote_addr":"[fd00:1122:3344:101::b]:60989","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":34,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:50:25.121933212Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:39459"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:50:25.122529156Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"8f73be18-b4f4-4026-a7c8-f2a1bfdfb555","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:39459","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":519,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:50:25.125828054Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"279a65bd-a44d-421e-a8ec-522974b99329","remote_addr":"[fd00:1122:3344:101::b]:60989","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":78,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:50:25.269657649Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:45209"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:50:25.269791043Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3824"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:50:25.294697254Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3835"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:50:25.318557201Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"834d3142-399d-4128-9a64-ba1120369888","remote_addr":"[fd00:1122:3344:101::c]:45209","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":48846,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:50:25.318718465Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:60019"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:50:25.318846157Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3824"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:50:25.343525304Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3835"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:50:25.365666849Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"554d9f51-6aa0-451d-a2a0-3e8b6426f3c1","remote_addr":"[fd00:1122:3344:101::b]:60019","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":46875,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:50:25.365987557Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"b723fb7f-bc22-4a70-8f03-350a31b763ee","remote_addr":"[fd00:1122:3344:101::c]:45209","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":138,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:50:25.36652127Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"3abf891f-bce2-4195-ab16-7d70f0a1712e","remote_addr":"[fd00:1122:3344:101::b]:60019","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":629,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:50:43.055188034Z","hostname":"buskin","pid":1555,"uri":"/8fa4bc78-1026-41f0-b09d-c67224071376","method":"GET","req_id":"4db7a64e-505e-400c-8165-5ca67e50b8b9","remote_addr":"[fd00:1122:3344:101::d]:46986","local_addr":"[fd00:1122:3344:101::1]:57429","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","component":"producer-server","component":"MetricsManager","sled_id":"8fa4bc78-1026-41f0-b09d-c67224071376","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":876,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:50:54.385209126Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:46311"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:50:54.385311709Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"8b37d6e2-7f04-456d-b41e-c04d515f9c16","remote_addr":"[fd00:1122:3344:101::a]:46311","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":98,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:50:54.391843984Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"400191f8-745d-47d2-b34b-90771b578a81","remote_addr":"[fd00:1122:3344:101::a]:46311","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":123,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:50:54.503123736Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:45342"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:50:54.5032925Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"551e6f4d-52ea-49a6-b0c3-8e92dd9b92a4","remote_addr":"[fd00:1122:3344:101::a]:45342","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":146,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:50:54.694137932Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:34741"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:50:54.694232704Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3824"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:50:54.719630767Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3835"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:50:54.74672284Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"9cf65a7c-44d4-4e00-a249-e8200b4f2450","remote_addr":"[fd00:1122:3344:101::a]:34741","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":52530,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:50:54.747058678Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"aabd8317-dcdb-4a09-b455-183103600b86","remote_addr":"[fd00:1122:3344:101::a]:34741","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":100,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:50:54.933727111Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:55582"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:50:54.933914955Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"56c44973-3848-4317-95f7-5665a8e2453b","remote_addr":"[fd00:1122:3344:101::b]:55582","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":168,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:50:54.939635841Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:40475"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:50:54.939734544Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"233779a1-2033-4c09-971a-6d41cdbd4266","remote_addr":"[fd00:1122:3344:101::b]:40475","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":55,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:50:54.946721379Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"0eaf7b10-667e-4dc0-9353-57c0b85c399c","remote_addr":"[fd00:1122:3344:101::b]:40475","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":93,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:50:54.985264355Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:52131"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:50:54.985366217Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"87efa877-2567-4edf-ba5b-fe6b0af6f2e7","remote_addr":"[fd00:1122:3344:101::c]:52131","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":56,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:50:54.992807664Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"43cc8c62-b54d-4e7f-a88c-eae5ed77f545","remote_addr":"[fd00:1122:3344:101::c]:52131","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":709,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:50:55.000857095Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:53589"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:50:55.000990318Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"42fd33d9-7fbb-433f-a7dd-9e8899d67699","remote_addr":"[fd00:1122:3344:101::c]:53589","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":95,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:50:55.071494773Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:64054"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:50:55.071611665Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3824"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:50:55.098163946Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3835"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:50:55.121283275Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"d4e980a6-6ddf-4ba4-8c07-375d686cd633","remote_addr":"[fd00:1122:3344:101::b]:64054","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":49720,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:50:55.121539011Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:64347"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:50:55.121572002Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"8060904a-6329-42ab-8b07-38eb0de96d9d","remote_addr":"[fd00:1122:3344:101::b]:64054","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":112,"response_code":"200"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:50:55.12193499Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3824"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:50:55.147304263Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3835"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:50:55.155738793Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:50595"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:50:55.155892447Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"53d1819c-995a-4cde-9b94-5499503bd698","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:50595","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":86,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:50:55.169830608Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"dc0cbbe9-a791-4e06-bbd7-3e270c30788d","remote_addr":"[fd00:1122:3344:101::c]:64347","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":47977,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:50:55.170183016Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"9a7a6078-3cee-4acf-a0d4-255659df6316","remote_addr":"[fd00:1122:3344:101::c]:64347","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":125,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:51:13.055239572Z","hostname":"buskin","pid":1555,"uri":"/8fa4bc78-1026-41f0-b09d-c67224071376","method":"GET","req_id":"a0d5f363-c958-4879-8831-fae3eb6b4152","remote_addr":"[fd00:1122:3344:101::d]:46986","local_addr":"[fd00:1122:3344:101::1]:57429","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","component":"producer-server","component":"MetricsManager","sled_id":"8fa4bc78-1026-41f0-b09d-c67224071376","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":572,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:51:24.481021416Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:37476"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:51:24.481245912Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"20b70580-b139-42e6-a01d-3d5fe4e1579b","remote_addr":"[fd00:1122:3344:101::a]:37476","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":206,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:51:24.627794281Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:42944"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:51:24.627908064Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"3cb0c9f1-e2a3-43a2-af58-205bc6d28cb2","remote_addr":"[fd00:1122:3344:101::a]:42944","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":48,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:51:24.635670248Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"5a538f53-1bfc-4fb3-b26d-e8e2d6c1a68d","remote_addr":"[fd00:1122:3344:101::a]:42944","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":111,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:51:24.935455147Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:46747"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:51:24.936749958Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3824"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:51:24.962777726Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3835"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:51:24.985889664Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"f2a2c44c-6e19-440b-bbda-3e81cc5cdaaa","remote_addr":"[fd00:1122:3344:101::a]:46747","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":50391,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:51:24.986257033Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"102a6627-eda4-4e6a-a380-627a77caed09","remote_addr":"[fd00:1122:3344:101::a]:46747","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":115,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:51:25.172867315Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:53364"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:51:25.173069449Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"b45f15a3-caaf-4281-9410-4024b0932ae7","remote_addr":"[fd00:1122:3344:101::b]:53364","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":144,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:51:25.174960214Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:32952"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:51:25.175029446Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"695e02a5-3bd5-43d0-8229-17bd4c138ad2","remote_addr":"[fd00:1122:3344:101::c]:32952","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":30,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:51:25.178402216Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:49745"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:51:25.178533199Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"2a8d442d-7057-4832-9f40-7d47189dcf45","remote_addr":"[fd00:1122:3344:101::c]:49745","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":79,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:51:25.183456576Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"1796e1d4-41a0-4fbe-a9a7-f50347daec02","remote_addr":"[fd00:1122:3344:101::c]:32952","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":83,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:51:25.190495003Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:52732"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:51:25.190571235Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"155cd9e1-6f41-4f19-89fd-c5c0b8b4082c","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:52732","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":27,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:51:25.28637783Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:50282"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:51:25.286455292Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"86fe111e-ed02-4ba2-8968-aa97fefe8125","remote_addr":"[fd00:1122:3344:101::b]:50282","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":47,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:51:25.295458266Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"ceffa0fe-1ddb-44a1-aba6-aee746492c5e","remote_addr":"[fd00:1122:3344:101::b]:50282","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":862,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:51:25.337673358Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:57404"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:51:25.33775522Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3824"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:51:25.36300586Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3835"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:51:25.387540872Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"965ef44d-7bd1-4058-aec7-06be58808d2c","remote_addr":"[fd00:1122:3344:101::c]:57404","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":49838,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:51:25.387779928Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"207cd807-8e17-4111-97fa-29be1d9b5ec5","remote_addr":"[fd00:1122:3344:101::c]:57404","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":85,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:51:25.477529729Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:52796"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:51:25.477736924Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3824"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:51:25.500468244Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3835"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:51:25.523140552Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"cb5b14c4-c507-4418-bb51-9f5edaf02af8","remote_addr":"[fd00:1122:3344:101::b]:52796","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":45477,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:51:25.523541422Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"a69c5a91-bb8f-4c52-ab40-8ebfa7769531","remote_addr":"[fd00:1122:3344:101::b]:52796","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":66,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:51:43.055741538Z","hostname":"buskin","pid":1555,"uri":"/8fa4bc78-1026-41f0-b09d-c67224071376","method":"GET","req_id":"f24b5984-10aa-4ca5-a9ce-1dcfe993e687","remote_addr":"[fd00:1122:3344:101::d]:46986","local_addr":"[fd00:1122:3344:101::1]:57429","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","component":"producer-server","component":"MetricsManager","sled_id":"8fa4bc78-1026-41f0-b09d-c67224071376","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":854,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:51:54.412474473Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:60944"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:51:54.412589296Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"1f1c22c3-b293-4f6d-9606-af42b33e5eb0","remote_addr":"[fd00:1122:3344:101::a]:60944","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":108,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:51:54.419224523Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"9dcbdeef-b26e-4a41-8b8a-16052a232c50","remote_addr":"[fd00:1122:3344:101::a]:60944","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":123,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:51:54.507858078Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:57271"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:51:54.508053232Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"8c866a1d-2611-4f2d-b50b-e0063d96c136","remote_addr":"[fd00:1122:3344:101::a]:57271","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":168,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:51:54.633381998Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:57565"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:51:54.63348085Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3824"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:51:54.659803486Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3835"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:51:54.684475881Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"9f921910-fb92-4a8d-80cb-7cce7c7b6f51","remote_addr":"[fd00:1122:3344:101::a]:57565","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":51037,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:51:54.685365441Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"3e15e7cb-1108-48ce-921f-8b20fc81a350","remote_addr":"[fd00:1122:3344:101::a]:57565","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":724,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:51:54.888319442Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:34749"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:51:54.888483656Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"fc4be165-3c0d-42a1-a9d9-5193a81a1237","remote_addr":"[fd00:1122:3344:101::c]:34749","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":110,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:51:54.894420027Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:46493"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:51:54.894511399Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"a99ee55e-28fe-4428-9566-be8e175f0c5d","remote_addr":"[fd00:1122:3344:101::b]:46493","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":57,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:51:54.895902752Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"e55456e7-3c68-4e98-9848-7f5ca52c09cd","remote_addr":"[fd00:1122:3344:101::c]:34749","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":130,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:51:54.902260813Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"b854c055-139d-454b-a115-8a1f6147561d","remote_addr":"[fd00:1122:3344:101::b]:46493","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":792,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:51:54.965982486Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:61748"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:51:54.96613683Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"15b9be5f-3814-4819-a2fc-10efee943793","remote_addr":"[fd00:1122:3344:101::b]:61748","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":135,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:51:54.978557215Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:43589"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:51:54.978708468Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"149a1511-2fc6-474e-a15f-5fa1f48f8549","remote_addr":"[fd00:1122:3344:101::c]:43589","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":103,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:51:55.075667131Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:35983"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:51:55.075720292Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3824"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:51:55.102385185Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3835"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:51:55.124520991Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"63822523-1187-43f7-a6f0-6ab4cc72df68","remote_addr":"[fd00:1122:3344:101::c]:35983","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":48816,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:51:55.124776687Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:54176"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:51:55.125874883Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3824"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:51:55.1303828Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"67036130-a74e-4e5c-a904-36a54225370d","remote_addr":"[fd00:1122:3344:101::c]:35983","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":5480,"response_code":"200"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:51:55.150716353Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3835"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:51:55.171928607Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"d790ed48-d99d-4925-89dc-7eda38151522","remote_addr":"[fd00:1122:3344:101::b]:54176","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":47030,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:51:55.172224624Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"3ae23da7-3d24-4ab8-8577-fe7dff2011a8","remote_addr":"[fd00:1122:3344:101::b]:54176","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":114,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:51:55.223924771Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:59794"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:51:55.224000233Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"90839fa5-81af-46ba-882a-5af129f43e07","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:59794","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":44,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:52:13.063123008Z","hostname":"buskin","pid":1555,"uri":"/8fa4bc78-1026-41f0-b09d-c67224071376","method":"GET","req_id":"e7d26e23-1e7d-4667-9243-d3250391ced3","remote_addr":"[fd00:1122:3344:101::d]:46986","local_addr":"[fd00:1122:3344:101::1]:57429","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","component":"producer-server","component":"MetricsManager","sled_id":"8fa4bc78-1026-41f0-b09d-c67224071376","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":295,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:52:24.459065375Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:52242"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:52:24.459196368Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"c96e8857-e351-4e95-8fa3-d38f5285f335","remote_addr":"[fd00:1122:3344:101::a]:52242","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":119,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:52:24.466572613Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"f9fcad9d-3e95-4e43-abfa-7ca571a0d1b6","remote_addr":"[fd00:1122:3344:101::a]:52242","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":132,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:52:24.50896735Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:49272"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:52:24.509112673Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"e9b493bd-d4db-411d-a340-3273f7ccf5d8","remote_addr":"[fd00:1122:3344:101::a]:49272","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":116,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:52:24.812251181Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:38869"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:52:24.812439006Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3824"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:52:24.840854901Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3835"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:52:24.86483052Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"faf9e2ca-a68f-4323-9f6e-ed3635f6dc7b","remote_addr":"[fd00:1122:3344:101::a]:38869","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":52490,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:52:24.865117687Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"51793651-9992-441b-be23-0322388aef88","remote_addr":"[fd00:1122:3344:101::a]:38869","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":106,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:52:24.935521669Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:50231"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:52:24.935648762Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"437b1c00-aba4-4bb1-9844-e34f3fdff33f","remote_addr":"[fd00:1122:3344:101::c]:50231","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":73,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:52:24.94355814Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"3724562c-dd55-4296-9392-e9dcdabe4c3c","remote_addr":"[fd00:1122:3344:101::c]:50231","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":757,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:52:24.969047725Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:42393"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:52:24.969174858Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"b8e76206-8523-40d2-88f7-1728d1a621b4","remote_addr":"[fd00:1122:3344:101::c]:42393","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":106,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:52:25.219299447Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:63017"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:52:25.219360189Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"d970257e-7c88-4f00-99cb-97f075dbdaf1","remote_addr":"[fd00:1122:3344:101::b]:63017","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":33,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:52:25.228608629Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"e9dc0de2-858a-4003-930e-26e2425dc4df","remote_addr":"[fd00:1122:3344:101::b]:63017","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":1053,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:52:25.242907648Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:35756"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:52:25.243050591Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"56415777-46a0-4d2e-9c7c-1cbb41b84f15","remote_addr":"[fd00:1122:3344:101::b]:35756","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":103,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:52:25.255805324Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:48361"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:52:25.255863666Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"daa22b88-54b1-4124-b327-6ff7baa8e189","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:48361","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":28,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:52:25.405186022Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:50384"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:52:25.405263663Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3824"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:52:25.432387157Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3835"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:52:25.456579832Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"ca45c504-57cb-4338-8401-e477c0b8a70a","remote_addr":"[fd00:1122:3344:101::c]:50384","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":51352,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:52:25.456814668Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:35708"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:52:25.456872159Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"b996e44e-2934-46f9-acad-3477ddc4c6df","remote_addr":"[fd00:1122:3344:101::c]:50384","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":108,"response_code":"200"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:52:25.456977841Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3824"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:52:25.482150359Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3835"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:52:25.505209057Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"ee095070-0e13-4e27-84f9-df1aed12ffbb","remote_addr":"[fd00:1122:3344:101::b]:35708","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":48293,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:52:25.505513874Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"f0db29b8-fe21-4d74-ab5e-84aadec9a86e","remote_addr":"[fd00:1122:3344:101::b]:35708","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":108,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:52:43.104531802Z","hostname":"buskin","pid":1555,"uri":"/8fa4bc78-1026-41f0-b09d-c67224071376","method":"GET","req_id":"0d0b7ff4-c485-421d-acf7-ad866c210521","remote_addr":"[fd00:1122:3344:101::d]:46986","local_addr":"[fd00:1122:3344:101::1]:57429","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","component":"producer-server","component":"MetricsManager","sled_id":"8fa4bc78-1026-41f0-b09d-c67224071376","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":569,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:52:54.412642886Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:33169"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:52:54.412741768Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"2e6efd11-7fca-4dd1-a19d-7315aecbbda9","remote_addr":"[fd00:1122:3344:101::a]:33169","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":92,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:52:54.419408226Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"b7195479-cd67-4d06-ac70-a66b55c2bb78","remote_addr":"[fd00:1122:3344:101::a]:33169","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":93,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:52:54.553770797Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:59647"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:52:54.553974101Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"70b08669-7008-40f2-9e93-e1095f9ad0c8","remote_addr":"[fd00:1122:3344:101::a]:59647","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":173,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:52:54.612407219Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:48284"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:52:54.613072345Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3824"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:52:54.639276637Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3835"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:52:54.662471778Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"a49b39bb-bf82-4c04-97ef-af68ac0ff3e7","remote_addr":"[fd00:1122:3344:101::a]:48284","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":50023,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:52:54.662723004Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"680e69ba-86cf-4a3e-ba78-4f674b57021d","remote_addr":"[fd00:1122:3344:101::a]:48284","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":94,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:52:54.882765079Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:57067"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:52:54.882929223Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"0c620675-f0f1-49ca-b467-7d47ca846c5b","remote_addr":"[fd00:1122:3344:101::c]:57067","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":110,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:52:54.890595185Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"5c2f6c4c-4df7-41a4-858c-e12868a23b33","remote_addr":"[fd00:1122:3344:101::c]:57067","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":137,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:52:54.901050783Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:47953"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:52:54.902042317Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"7f2d7006-7964-452a-b5cd-342025084ea3","remote_addr":"[fd00:1122:3344:101::c]:47953","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":970,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:52:54.95187273Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:44737"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:52:54.951952382Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"ae1a1570-3a7c-4b47-bc71-325d78e96acb","remote_addr":"[fd00:1122:3344:101::b]:44737","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":41,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:52:54.960117696Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"20d31239-1aa4-4157-bb51-e3a81c69ea03","remote_addr":"[fd00:1122:3344:101::b]:44737","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":80,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:52:55.0301966Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:46958"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:52:55.031077001Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"535254e1-4f09-4ac2-97be-69ebdeddfcdc","remote_addr":"[fd00:1122:3344:101::b]:46958","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":838,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:52:55.090246536Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:50297"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:52:55.090319538Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3824"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:52:55.115477555Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3835"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:52:55.139710461Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"25a0b27c-c267-4b1c-af1a-6194bfa7d8c1","remote_addr":"[fd00:1122:3344:101::c]:50297","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":49414,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:52:55.139859834Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:39281"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:52:55.141105754Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3824"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:52:55.164423878Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3835"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:52:55.18598948Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"49975b0e-c2ba-4451-93c2-299f7f7370a4","remote_addr":"[fd00:1122:3344:101::b]:39281","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":45518,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:52:55.186372179Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"a2f7ec23-3b5d-400f-ad5f-167a04d96775","remote_addr":"[fd00:1122:3344:101::c]:50297","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":179,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:52:55.186391879Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"075f6059-a15d-4b12-83bd-18022641d15f","remote_addr":"[fd00:1122:3344:101::b]:39281","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":146,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:52:55.289759074Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:43888"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:52:55.289867096Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"945015b9-2b6a-443b-96dd-06d1892e3494","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:43888","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":65,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:13.106209654Z","hostname":"buskin","pid":1555,"uri":"/8fa4bc78-1026-41f0-b09d-c67224071376","method":"GET","req_id":"fcd85374-cebd-4fea-a747-ac2ac2c64c52","remote_addr":"[fd00:1122:3344:101::d]:46986","local_addr":"[fd00:1122:3344:101::1]:57429","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","component":"producer-server","component":"MetricsManager","sled_id":"8fa4bc78-1026-41f0-b09d-c67224071376","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":2410,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:24.506987807Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:44990"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:24.50793984Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"03883dee-6218-4d4f-93c7-7d521802cfd5","remote_addr":"[fd00:1122:3344:101::a]:44990","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":933,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:24.666351641Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:63644"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:24.666632138Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:673","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: a6846ef5-8686-4c7e-ae28-d178a4029a85, kind: Service { id: 2c3f8453-233f-428f-93cc-0bfefb84929d }, name: Name(\"external-dns-2c3f8453-233f-428f-93cc-0bfefb84929d\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 164, 190])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: f04fae05-ec27-43ac-ad4d-ab63c091bf1b, kind: Service { id: b04db43b-dc24-43ce-9d97-32193f88e046 }, name: Name(\"external-dns-b04db43b-dc24-43ce-9d97-32193f88e046\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 154])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 735e2401-2ba5-4079-b4a0-7f0184716575, kind: Service { id: e4efbc31-e513-4290-82f5-c51d8afea0c7 }, name: Name(\"nexus-e4efbc31-e513-4290-82f5-c51d8afea0c7\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 154])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 9e8859b7-3d6e-4066-b58c-24fc535c1122, kind: Service { id: 221c5e7c-0832-4890-8b0a-f9ba08725876 }, name: Name(\"nexus-221c5e7c-0832-4890-8b0a-f9ba08725876\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 155])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: f33554f0-0c2d-4b66-98d0-73365d475c4b, kind: Service { id: 85472347-dfd9-425e-bd98-9b81ebdf3b2b }, name: Name(\"nexus-85472347-dfd9-425e-bd98-9b81ebdf3b2b\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 244])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:24.666679159Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:24.66673383Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:24.666764241Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:24.666794152Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:24.666834833Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:24.666861593Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:24.666892584Z","hostname":"buskin","pid":1555,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"27c16381-f3c2-4e8e-9e81-3ee6204fcf72","remote_addr":"[fd00:1122:3344:101::a]:63644","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":480,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:24.698499334Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:62627"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:24.698581196Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"02c98f30-2b1f-49d7-93a2-38a6b2c15861","remote_addr":"[fd00:1122:3344:101::a]:62627","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":38,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:24.705145392Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"cdbd03f9-2fa7-49ee-b542-ba0beef7605a","remote_addr":"[fd00:1122:3344:101::a]:62627","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":79,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:24.872025245Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:33141"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:24.872135508Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3824"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:24.901476534Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3835"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:24.922590406Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"af6c66ab-32e1-4fc6-8f9d-aa0fb7aa428d","remote_addr":"[fd00:1122:3344:101::a]:33141","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":50516,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:24.92402119Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"17847ee7-9aa8-41e1-9ec7-c10daeaaccd7","remote_addr":"[fd00:1122:3344:101::a]:33141","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":1273,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:24.993590862Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:63442"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:24.993866518Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"979d09ba-a2e9-4a9c-8637-aff4871a46de","remote_addr":"[fd00:1122:3344:101::c]:63442","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":210,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:25.009240524Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:43533"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:25.009308885Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"32ca88d5-f6dc-46bc-a5c8-cb5cc881cfc8","remote_addr":"[fd00:1122:3344:101::c]:43533","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":42,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:25.016009944Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"54839fe0-178e-41bb-ad03-615edb66b842","remote_addr":"[fd00:1122:3344:101::c]:43533","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":110,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:25.175406689Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:45384"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:25.17545786Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"e886a522-5db9-448a-9ece-467eeabdb90a","remote_addr":"[fd00:1122:3344:101::b]:45384","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":26,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:25.183975993Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"518acf99-53c1-4b1f-bd6b-6682b56bd792","remote_addr":"[fd00:1122:3344:101::b]:45384","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":853,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:25.213628777Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:47692"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:25.21376623Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:673","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: a6846ef5-8686-4c7e-ae28-d178a4029a85, kind: Service { id: 2c3f8453-233f-428f-93cc-0bfefb84929d }, name: Name(\"external-dns-2c3f8453-233f-428f-93cc-0bfefb84929d\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 164, 190])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: f04fae05-ec27-43ac-ad4d-ab63c091bf1b, kind: Service { id: b04db43b-dc24-43ce-9d97-32193f88e046 }, name: Name(\"external-dns-b04db43b-dc24-43ce-9d97-32193f88e046\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 154])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 735e2401-2ba5-4079-b4a0-7f0184716575, kind: Service { id: e4efbc31-e513-4290-82f5-c51d8afea0c7 }, name: Name(\"nexus-e4efbc31-e513-4290-82f5-c51d8afea0c7\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 154])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 9e8859b7-3d6e-4066-b58c-24fc535c1122, kind: Service { id: 221c5e7c-0832-4890-8b0a-f9ba08725876 }, name: Name(\"nexus-221c5e7c-0832-4890-8b0a-f9ba08725876\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 155])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: f33554f0-0c2d-4b66-98d0-73365d475c4b, kind: Service { id: 85472347-dfd9-425e-bd98-9b81ebdf3b2b }, name: Name(\"nexus-85472347-dfd9-425e-bd98-9b81ebdf3b2b\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 244])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:25.213799811Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:25.213832642Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:25.213845552Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:25.213867463Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:25.213899824Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:25.213916184Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:25.213926894Z","hostname":"buskin","pid":1555,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"ca2d3949-a760-473b-ace0-8c1e324efc0f","remote_addr":"[fd00:1122:3344:101::c]:47692","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":285,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:25.244692585Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:65472"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:25.244859039Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"90f459fd-246b-495a-aefa-8202520b1953","remote_addr":"[fd00:1122:3344:101::b]:65472","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":121,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:25.262772304Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:47977"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:25.262844726Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3824"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:25.286261462Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3835"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:25.307636669Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"cedc10aa-d7cf-4703-b930-c25f6d75ed97","remote_addr":"[fd00:1122:3344:101::c]:47977","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":44815,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:25.307939337Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"b6f0d70c-82d8-4205-a6d7-d6dbd2adb8fc","remote_addr":"[fd00:1122:3344:101::c]:47977","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":104,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:25.327908811Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:61702"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:25.328593837Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"098fb54e-2b97-41a0-a333-03f297fef238","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:61702","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":622,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:25.417770025Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:37281"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:25.41801232Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:673","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: a6846ef5-8686-4c7e-ae28-d178a4029a85, kind: Service { id: 2c3f8453-233f-428f-93cc-0bfefb84929d }, name: Name(\"external-dns-2c3f8453-233f-428f-93cc-0bfefb84929d\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 164, 190])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: f04fae05-ec27-43ac-ad4d-ab63c091bf1b, kind: Service { id: b04db43b-dc24-43ce-9d97-32193f88e046 }, name: Name(\"external-dns-b04db43b-dc24-43ce-9d97-32193f88e046\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 194, 154])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 735e2401-2ba5-4079-b4a0-7f0184716575, kind: Service { id: e4efbc31-e513-4290-82f5-c51d8afea0c7 }, name: Name(\"nexus-e4efbc31-e513-4290-82f5-c51d8afea0c7\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 205, 154])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 9e8859b7-3d6e-4066-b58c-24fc535c1122, kind: Service { id: 221c5e7c-0832-4890-8b0a-f9ba08725876 }, name: Name(\"nexus-221c5e7c-0832-4890-8b0a-f9ba08725876\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 245, 155])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: f33554f0-0c2d-4b66-98d0-73365d475c4b, kind: Service { id: 85472347-dfd9-425e-bd98-9b81ebdf3b2b }, name: Name(\"nexus-85472347-dfd9-425e-bd98-9b81ebdf3b2b\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 150, 244])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:25.418052931Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:25.418096442Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:25.418123643Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:25.418140414Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:25.418155404Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:25.418177024Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:25.418194505Z","hostname":"buskin","pid":1555,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"7125b5ad-d795-45b5-82b3-d1a09c1af32c","remote_addr":"[fd00:1122:3344:101::b]:37281","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":362,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:25.488426252Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:62767"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:25.488510754Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3824"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:25.512802391Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3835"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:25.535229354Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"a79409b0-5c02-4985-a54a-053bf55e5a20","remote_addr":"[fd00:1122:3344:101::b]:62767","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":46752,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:25.536174546Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"d92f3214-c0f8-4261-9f5b-9ad98c27e709","remote_addr":"[fd00:1122:3344:101::b]:62767","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":745,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:43.106203323Z","hostname":"buskin","pid":1555,"uri":"/8fa4bc78-1026-41f0-b09d-c67224071376","method":"GET","req_id":"4ed7b1a9-d119-4563-a1cf-484cfae56783","remote_addr":"[fd00:1122:3344:101::d]:46986","local_addr":"[fd00:1122:3344:101::1]:57429","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","component":"producer-server","component":"MetricsManager","sled_id":"8fa4bc78-1026-41f0-b09d-c67224071376","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":2256,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:54.522024603Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:33607"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:54.522130605Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"86d6df89-d3f6-4a1d-acb4-be177928eda3","remote_addr":"[fd00:1122:3344:101::a]:33607","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":100,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:54.529575622Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"e9711708-3c0f-4cb3-80ea-0744635d06f3","remote_addr":"[fd00:1122:3344:101::a]:33607","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":128,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:54.532830059Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:54998"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:54.532962123Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"a6a49b74-3598-4926-a08f-086a443e606b","remote_addr":"[fd00:1122:3344:101::a]:54998","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":101,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:54.65878688Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:58549"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:54.658916353Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3824"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:54.682948194Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3835"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:54.704710931Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"b66ba3f9-43b7-4452-9d47-72723b7a3113","remote_addr":"[fd00:1122:3344:101::a]:58549","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":45879,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:54.705504539Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"c279c803-6102-4f1a-ae65-3f2453eb6b63","remote_addr":"[fd00:1122:3344:101::a]:58549","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":646,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:54.87102109Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:56750"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:54.871165243Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"4821c2b5-2983-446d-a12d-ae0278be15b4","remote_addr":"[fd00:1122:3344:101::c]:56750","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":94,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:54.877843132Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"d80fcd83-c0a8-4f10-8ea1-1d99eb496013","remote_addr":"[fd00:1122:3344:101::c]:56750","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":135,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:54.910724332Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:46483"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:54.910784234Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"1df3f34e-6efe-45ea-95a8-84dec249cef2","remote_addr":"[fd00:1122:3344:101::b]:46483","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":44,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:54.91778938Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"5b89a462-7309-4db2-81e1-32078e5bc38f","remote_addr":"[fd00:1122:3344:101::b]:46483","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":90,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:54.970132513Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:45063"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:54.970306127Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"f179eb8b-bb9f-42e3-b3d9-d37ca15181e7","remote_addr":"[fd00:1122:3344:101::c]:45063","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":128,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:55.082250206Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:54018"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:55.082365099Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"28f944dd-85b7-4e38-95f2-cc4c30e99b79","remote_addr":"[fd00:1122:3344:101::b]:54018","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":95,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:55.129566179Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:43364"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:55.129621001Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3824"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:55.154345498Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3835"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:55.176522664Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"7322b6c5-0a24-4cb6-b367-6bde9dff1fb4","remote_addr":"[fd00:1122:3344:101::c]:43364","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":46920,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:55.176680548Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:51513"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:55.176899563Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3824"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:55.180294084Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"d5767862-466c-4adc-828e-5d588b9d62bf","remote_addr":"[fd00:1122:3344:101::c]:43364","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":3393,"response_code":"200"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:55.200174436Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3835"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:55.221748278Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"5a623333-5801-419e-907f-dc2b2f4b9d3e","remote_addr":"[fd00:1122:3344:101::b]:51513","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":44968,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:55.222518207Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"439cd6b6-68ce-4b37-ba30-789c632f71ea","remote_addr":"[fd00:1122:3344:101::b]:51513","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":587,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:55.362590893Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:44818"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:53:55.363086545Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"9b51407e-4f1f-4479-96fb-6c1100f410fe","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:44818","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":468,"response_code":"200"} {"msg":"running automatic periodic zone bundle cleanup","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:54:01.848015385Z","hostname":"buskin","pid":1555,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:303"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:54:01.848098557Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:54:01.884478931Z","hostname":"buskin","pid":1555,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:292","time_to_next_cleanup":"599.99999628s","next_cleanup":"Instant { tv_sec: 6804, tv_nsec: 917992695 }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:54:13.106230992Z","hostname":"buskin","pid":1555,"uri":"/8fa4bc78-1026-41f0-b09d-c67224071376","method":"GET","req_id":"246c1840-60a5-4722-86a1-fe4018aa6d3d","remote_addr":"[fd00:1122:3344:101::d]:46986","local_addr":"[fd00:1122:3344:101::1]:57429","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","component":"producer-server","component":"MetricsManager","sled_id":"8fa4bc78-1026-41f0-b09d-c67224071376","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":1655,"response_code":"200"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:54:15.69441576Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:54:24.473266173Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:53741"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:54:24.473485528Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"d57c38e4-7925-4031-af48-a7c65a24f809","remote_addr":"[fd00:1122:3344:101::a]:53741","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":203,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:54:24.492192412Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:51483"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:54:24.49291413Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"bce7018b-4022-46c1-962e-c494477aeda7","remote_addr":"[fd00:1122:3344:101::a]:51483","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":700,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:54:24.500813257Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"805d3537-9ae5-4fc5-97b2-a7fac3143244","remote_addr":"[fd00:1122:3344:101::a]:51483","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":109,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:54:24.875225218Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:61066"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:54:24.87533763Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3824"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:54:24.906019829Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3835"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:54:24.932517348Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"74ec9cd3-2b0c-4220-8350-1c9799997534","remote_addr":"[fd00:1122:3344:101::a]:61066","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":57241,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:54:24.938458959Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"59f79cf1-0708-4bc4-a777-1ab6466b88d3","remote_addr":"[fd00:1122:3344:101::a]:61066","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":101,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:54:24.982075925Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:43114"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:54:24.982336621Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"8cf7d59e-c974-4575-b871-86481ce6ca3a","remote_addr":"[fd00:1122:3344:101::c]:43114","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":185,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:54:25.083482393Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:51756"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:54:25.083560485Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"ba9a1915-89c3-4648-aef7-dee76ad53574","remote_addr":"[fd00:1122:3344:101::b]:51756","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":48,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:54:25.091079093Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"bb942a3c-df96-4f11-ac7c-01eae461d709","remote_addr":"[fd00:1122:3344:101::b]:51756","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":110,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:54:25.227473672Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:58834"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:54:25.228449326Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"fd6552b2-aa0b-4cd9-988a-8419d5d16c27","remote_addr":"[fd00:1122:3344:101::b]:58834","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":929,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:54:25.276065926Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:55362"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:54:25.276184189Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"bd5ee10e-62a0-48e1-b4c0-fff04d515da3","remote_addr":"[fd00:1122:3344:101::c]:55362","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":102,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:54:25.285712586Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"bbd538da-3a96-4fcd-a95a-3f7214c78c38","remote_addr":"[fd00:1122:3344:101::c]:55362","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":840,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:54:25.392564133Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:45776"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:54:25.392665895Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3824"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:54:25.426338485Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3835"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:54:25.455105298Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"657462c6-6f0a-425b-b397-ee6da91c7df0","remote_addr":"[fd00:1122:3344:101::c]:45776","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":62488,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:54:25.455302553Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:65253"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:54:25.455460366Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:33948"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:54:25.455645191Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3824"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:54:25.459881441Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"d2ad83ea-3a16-40d8-8c4a-541f86aa7849","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:33948","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":4181,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:54:25.460058676Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"b051bfc7-67f8-4acb-858c-a03f4f546299","remote_addr":"[fd00:1122:3344:101::c]:45776","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":4439,"response_code":"200"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:54:25.486868052Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3835"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:54:25.514815896Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"fab19d2b-47b5-42e2-8009-8249d7f02a17","remote_addr":"[fd00:1122:3344:101::b]:65253","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":59337,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:54:25.515191335Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"72dacc4b-1d3f-4bc7-bb28-5aa9122f1b65","remote_addr":"[fd00:1122:3344:101::b]:65253","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":142,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:54:43.105188115Z","hostname":"buskin","pid":1555,"uri":"/8fa4bc78-1026-41f0-b09d-c67224071376","method":"GET","req_id":"c8bdbca9-4432-4e95-b1e9-e6ebb92892c2","remote_addr":"[fd00:1122:3344:101::d]:46986","local_addr":"[fd00:1122:3344:101::1]:57429","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","component":"producer-server","component":"MetricsManager","sled_id":"8fa4bc78-1026-41f0-b09d-c67224071376","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":637,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:54:54.607063478Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:35945"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:54:54.607876897Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"c94f3672-bdd0-4eba-9fdd-dd6e794221e0","remote_addr":"[fd00:1122:3344:101::a]:35945","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":782,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:54:54.614874463Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"8dbf994b-edd9-4d80-b289-95f7413440f7","remote_addr":"[fd00:1122:3344:101::a]:35945","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":129,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:54:54.620331483Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:48904"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:54:54.622201217Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"092e4ba2-9b91-4386-9c9d-3a3ef54deb5b","remote_addr":"[fd00:1122:3344:101::a]:48904","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":1831,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:54:54.814630167Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:50714"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:54:54.814722729Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3824"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:54:54.841616318Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3835"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:54:54.866298744Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"b2d6a866-3fe2-41b7-8668-e072ee48cc7d","remote_addr":"[fd00:1122:3344:101::a]:50714","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":51626,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:54:54.866596811Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:36193"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:54:54.866805166Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"8d373ce2-6af8-4445-bb45-4e8ac360c6c1","remote_addr":"[fd00:1122:3344:101::a]:50714","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":220,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:54:54.867661236Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"2b5a6355-f8d9-40ea-8ec9-80801bc1acaf","remote_addr":"[fd00:1122:3344:101::c]:36193","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":981,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:54:54.875014131Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"f193489d-b36c-4593-8bc5-4af0a5b97563","remote_addr":"[fd00:1122:3344:101::c]:36193","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":171,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:54:54.896380738Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:62496"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:54:54.89647266Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"ffd00c85-4f5a-4ef1-9ff2-3c7ee54da3b5","remote_addr":"[fd00:1122:3344:101::b]:62496","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":31,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:54:54.903226441Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:42442"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:54:54.904431969Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"539b6cc1-ffae-4440-bafb-c90d52ff940f","remote_addr":"[fd00:1122:3344:101::c]:42442","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":1145,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:54:54.904542892Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"2733d69a-6540-4c84-a6aa-68fe6b54ccc2","remote_addr":"[fd00:1122:3344:101::b]:62496","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":58,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:54:54.974065843Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:63997"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:54:54.974945684Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"e3f8500d-7cac-4a04-8665-3e1ee35aeffd","remote_addr":"[fd00:1122:3344:101::b]:63997","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":860,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:54:55.078031852Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:46128"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:54:55.078106423Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3824"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:54:55.104784917Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3835"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:54:55.128586772Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"c2b5ecf5-09cb-45ce-90a2-a713d43ac6e4","remote_addr":"[fd00:1122:3344:101::c]:46128","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":50512,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:54:55.128863589Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:56699"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:54:55.129674108Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3824"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:54:55.1297506Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"a8539df6-abc9-4b4c-bb47-ac34a2e84b19","remote_addr":"[fd00:1122:3344:101::c]:46128","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":887,"response_code":"200"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:54:55.157363685Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3835"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:54:55.183077956Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"70759d14-35ee-4af8-a520-4a45c027ce59","remote_addr":"[fd00:1122:3344:101::b]:56699","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":54084,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:54:55.183492616Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"4f42d3a7-2f40-46b5-a5b3-a942fcc7c43e","remote_addr":"[fd00:1122:3344:101::b]:56699","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":145,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:54:55.493577549Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:64593"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:54:55.494136353Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"18072aaa-0a5b-4f1c-aebb-142119778858","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:64593","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":523,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:03.281865939Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:49630"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:03.282568176Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"f238aaa5-fd50-4e4f-a7a8-dcfd791002e9","remote_addr":"[fd00:1122:3344:101::a]:49630","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":652,"response_code":"200"} {"msg":"Mapping virtual NIC to physical host","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:03.282854363Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:780","mapping":"VirtualNetworkInterfaceHost { virtual_ip: 172.30.0.5, virtual_mac: MacAddr(MacAddr6([168, 64, 37, 243, 50, 113])), physical_host_ip: fd00:1122:3344:101::1, vni: Vni(9219180) }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:03.282894033Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"PUT","req_id":"920ba4da-91bc-4afd-8d2a-e6c5b7e02a39","remote_addr":"[fd00:1122:3344:101::a]:49630","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":120,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:03.437951946Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:62676"} {"msg":"ensuring instance is registered","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:03.450745679Z","hostname":"buskin","pid":1555,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:577","metadata":"InstanceMetadata { silo_id: bb76b9e7-26a5-4e9e-a61c-3de62341dc71, project_id: 5b8f924c-3552-42d1-99ac-94cb0940fe4b }","propolis_addr":"[fd00:1122:3344:101::1:0]:12400","vmm_runtime":"VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-07-25T05:55:02.992660Z }","instance_runtime":"InstanceRuntimeState { propolis_id: Some(a11cf978-990f-48d4-8597-cbd622e7cdbe (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-07-25T05:55:03.025890951Z }","hardware":"InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: 21d50e0c-41c1-477c-ae5c-1216ff3e1c93, kind: Instance { id: 057c7948-d027-4c43-bcaf-62810b07fef3 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 243, 50, 113])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(9219180), primary: true, slot: 0, transit_ips: [] }], source_nat: SourceNatConfig { ip: 10.151.1.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.1.176), floating_ips: [], firewall_rules: [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 21d50e0c-41c1-477c-ae5c-1216ff3e1c93, kind: Instance { id: 057c7948-d027-4c43-bcaf-62810b07fef3 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 243, 50, 113])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(9219180), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 21d50e0c-41c1-477c-ae5c-1216ff3e1c93, kind: Instance { id: 057c7948-d027-4c43-bcaf-62810b07fef3 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 243, 50, 113])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(9219180), primary: true, slot: 0, transit_ips: [] }], filter_hosts: Some([Vpc(Vni(9219180))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 21d50e0c-41c1-477c-ae5c-1216ff3e1c93, kind: Instance { id: 057c7948-d027-4c43-bcaf-62810b07fef3 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 243, 50, 113])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(9219180), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-e05b56d916a\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 4b57adae-59ff-4471-a606-c565e10339d7, read_only_parent: Some(Volume { block_size: 512, id: d266589e-cb07-4700-8896-7ca6fc087b4e, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 2a120dbc-0998-48d4-bfb5-e87686f7cf0c, key: Some(\"tAz9C6olO0l7NhRkBocuHL03NyM/pboGlS5gY9ANX2s=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::19]:19001\", \"[fd00:1122:3344:101::15]:19002\", \"[fd00:1122:3344:101::17]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 4b57adae-59ff-4471-a606-c565e10339d7, key: Some(\"EYyu4JaQfaGj+EquIBBl+na9yNkcVzIJU1YGUxTiCks=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::17]:19002\", \"[fd00:1122:3344:101::1a]:19001\", \"[fd00:1122:3344:101::13]:19000\"] } }] } }], cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAACL4S75WPlYAADhLvlYAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAjOEu+Vj5WAAA4S75WAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiIwNTdjNzk0OC1kMDI3LTRjNDMtYmNhZi02MjgxMGIwN2ZlZjMiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJUDhENWJISjJLSUVxa2VTVm1YNUZ6Z21oMGVlbFU3MHpOY0p6bU1rbkgwNSJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\") }","propolis_id":"a11cf978-990f-48d4-8597-cbd622e7cdbe","instance_id":"057c7948-d027-4c43-bcaf-62810b07fef3"} {"msg":"registering new instance","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:03.450841692Z","hostname":"buskin","pid":1555,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:611","instance_id":"057c7948-d027-4c43-bcaf-62810b07fef3 (instance)"} {"msg":"initializing new Instance","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:03.452205484Z","hostname":"buskin","pid":1555,"component":"InstanceManager","file":"sled-agent/src/instance.rs:951","state":"InstanceInitialState { hardware: InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: 21d50e0c-41c1-477c-ae5c-1216ff3e1c93, kind: Instance { id: 057c7948-d027-4c43-bcaf-62810b07fef3 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 243, 50, 113])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(9219180), primary: true, slot: 0, transit_ips: [] }], source_nat: SourceNatConfig { ip: 10.151.1.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.1.176), floating_ips: [], firewall_rules: [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 21d50e0c-41c1-477c-ae5c-1216ff3e1c93, kind: Instance { id: 057c7948-d027-4c43-bcaf-62810b07fef3 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 243, 50, 113])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(9219180), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 21d50e0c-41c1-477c-ae5c-1216ff3e1c93, kind: Instance { id: 057c7948-d027-4c43-bcaf-62810b07fef3 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 243, 50, 113])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(9219180), primary: true, slot: 0, transit_ips: [] }], filter_hosts: Some([Vpc(Vni(9219180))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 21d50e0c-41c1-477c-ae5c-1216ff3e1c93, kind: Instance { id: 057c7948-d027-4c43-bcaf-62810b07fef3 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 243, 50, 113])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(9219180), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-e05b56d916a\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 4b57adae-59ff-4471-a606-c565e10339d7, read_only_parent: Some(Volume { block_size: 512, id: d266589e-cb07-4700-8896-7ca6fc087b4e, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 2a120dbc-0998-48d4-bfb5-e87686f7cf0c, key: Some(\"tAz9C6olO0l7NhRkBocuHL03NyM/pboGlS5gY9ANX2s=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::19]:19001\", \"[fd00:1122:3344:101::15]:19002\", \"[fd00:1122:3344:101::17]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 4b57adae-59ff-4471-a606-c565e10339d7, key: Some(\"EYyu4JaQfaGj+EquIBBl+na9yNkcVzIJU1YGUxTiCks=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::17]:19002\", \"[fd00:1122:3344:101::1a]:19001\", \"[fd00:1122:3344:101::13]:19000\"] } }] } }], cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAACL4S75WPlYAADhLvlYAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAjOEu+Vj5WAAA4S75WAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiIwNTdjNzk0OC1kMDI3LTRjNDMtYmNhZi02MjgxMGIwN2ZlZjMiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJUDhENWJISjJLSUVxa2VTVm1YNUZ6Z21oMGVlbFU3MHpOY0p6bU1rbkgwNSJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\") }, instance_runtime: InstanceRuntimeState { propolis_id: Some(a11cf978-990f-48d4-8597-cbd622e7cdbe (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-07-25T05:55:03.025890951Z }, vmm_runtime: VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-07-25T05:55:02.992660Z }, propolis_addr: [fd00:1122:3344:101::1:0]:12400 }","propolis_id":"a11cf978-990f-48d4-8597-cbd622e7cdbe","instance_id":"057c7948-d027-4c43-bcaf-62810b07fef3"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:03.45288037Z","hostname":"buskin","pid":1555,"uri":"/instances/057c7948-d027-4c43-bcaf-62810b07fef3","method":"PUT","req_id":"7e64c280-b774-4d64-a0ca-9ee515e85f91","remote_addr":"[fd00:1122:3344:101::a]:62676","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":14873,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:03.631588763Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:42469"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:03.632817432Z","hostname":"buskin","pid":1555,"uri":"/instances/057c7948-d027-4c43-bcaf-62810b07fef3/state","method":"PUT","req_id":"ef9f21e4-6e18-4f58-8d4e-40ecb7e87c25","remote_addr":"[fd00:1122:3344:101::a]:42469","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":1144,"response_code":"200"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:03.633794156Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:426","port":"Port { inner: PortInner(PortData { name: \"opte6\", ip: 172.30.0.5, mac: MacAddr6([168, 64, 37, 243, 50, 113]), slot: 0, vni: Vni { inner: 9219180 }, subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.0.1 } }) }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:03.634349929Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Configuring new Omicron zone: oxz_propolis-server_a11cf978-990f-48d4-8597-cbd622e7cdbe","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:03.670538998Z","hostname":"buskin","pid":1555,"instance_id":"057c7948-d027-4c43-bcaf-62810b07fef3","component":"InstanceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_propolis-server_a11cf978-990f-48d4-8597-cbd622e7cdbe","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:03.694182239Z","hostname":"buskin","pid":1555,"instance_id":"057c7948-d027-4c43-bcaf-62810b07fef3","component":"InstanceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_propolis-server_a11cf978-990f-48d4-8597-cbd622e7cdbe:\n\n\n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:05.022209185Z","hostname":"buskin","pid":1555,"instance_id":"057c7948-d027-4c43-bcaf-62810b07fef3","component":"InstanceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_propolis-server_a11cf978-990f-48d4-8597-cbd622e7cdbe zone","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:05.022455191Z","hostname":"buskin","pid":1555,"zone":"oxz_propolis-server_a11cf978-990f-48d4-8597-cbd622e7cdbe","instance_id":"057c7948-d027-4c43-bcaf-62810b07fef3","component":"InstanceManager","file":"illumos-utils/src/running_zone.rs:465"} {"msg":"wait for service Some(\"oxz_propolis-server_a11cf978-990f-48d4-8597-cbd622e7cdbe\") failed: Property not found. retry in 47.318565ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:55:05.220688648Z","hostname":"buskin","pid":1555,"zone":"oxz_propolis-server_a11cf978-990f-48d4-8597-cbd622e7cdbe","instance_id":"057c7948-d027-4c43-bcaf-62810b07fef3","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_a11cf978-990f-48d4-8597-cbd622e7cdbe\") failed: Property not found. retry in 85.301097ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:55:05.277048127Z","hostname":"buskin","pid":1555,"zone":"oxz_propolis-server_a11cf978-990f-48d4-8597-cbd622e7cdbe","instance_id":"057c7948-d027-4c43-bcaf-62810b07fef3","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_a11cf978-990f-48d4-8597-cbd622e7cdbe\") failed: Property not found. retry in 258.137467ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:55:05.378847184Z","hostname":"buskin","pid":1555,"zone":"oxz_propolis-server_a11cf978-990f-48d4-8597-cbd622e7cdbe","instance_id":"057c7948-d027-4c43-bcaf-62810b07fef3","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_a11cf978-990f-48d4-8597-cbd622e7cdbe\") failed: Property not found. retry in 290.194886ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:55:05.651185161Z","hostname":"buskin","pid":1555,"zone":"oxz_propolis-server_a11cf978-990f-48d4-8597-cbd622e7cdbe","instance_id":"057c7948-d027-4c43-bcaf-62810b07fef3","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_a11cf978-990f-48d4-8597-cbd622e7cdbe\") failed: Property not found. retry in 415.066175ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:55:05.952082836Z","hostname":"buskin","pid":1555,"zone":"oxz_propolis-server_a11cf978-990f-48d4-8597-cbd622e7cdbe","instance_id":"057c7948-d027-4c43-bcaf-62810b07fef3","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_a11cf978-990f-48d4-8597-cbd622e7cdbe\") failed: Property not found. retry in 576.853984ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:55:06.376872443Z","hostname":"buskin","pid":1555,"zone":"oxz_propolis-server_a11cf978-990f-48d4-8597-cbd622e7cdbe","instance_id":"057c7948-d027-4c43-bcaf-62810b07fef3","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_a11cf978-990f-48d4-8597-cbd622e7cdbe\") failed: Property not found. retry in 693.911549ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:55:06.965654814Z","hostname":"buskin","pid":1555,"zone":"oxz_propolis-server_a11cf978-990f-48d4-8597-cbd622e7cdbe","instance_id":"057c7948-d027-4c43-bcaf-62810b07fef3","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_a11cf978-990f-48d4-8597-cbd622e7cdbe\") failed: Property not found. retry in 779.486355ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:55:07.670272886Z","hostname":"buskin","pid":1555,"zone":"oxz_propolis-server_a11cf978-990f-48d4-8597-cbd622e7cdbe","instance_id":"057c7948-d027-4c43-bcaf-62810b07fef3","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_a11cf978-990f-48d4-8597-cbd622e7cdbe\") failed: Property not found. retry in 685.101643ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:55:08.460972202Z","hostname":"buskin","pid":1555,"zone":"oxz_propolis-server_a11cf978-990f-48d4-8597-cbd622e7cdbe","instance_id":"057c7948-d027-4c43-bcaf-62810b07fef3","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_a11cf978-990f-48d4-8597-cbd622e7cdbe\") failed: Property not found. retry in 1.171955963s","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:55:09.157307217Z","hostname":"buskin","pid":1555,"zone":"oxz_propolis-server_a11cf978-990f-48d4-8597-cbd622e7cdbe","instance_id":"057c7948-d027-4c43-bcaf-62810b07fef3","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Started propolis in zone: oxz_propolis-server_a11cf978-990f-48d4-8597-cbd622e7cdbe","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:10.358437139Z","hostname":"buskin","pid":1555,"instance_id":"057c7948-d027-4c43-bcaf-62810b07fef3","component":"InstanceManager","file":"sled-agent/src/instance.rs:1460"} {"msg":"wait for service Some(\"oxz_propolis-server_a11cf978-990f-48d4-8597-cbd622e7cdbe\") failed: Property not found. retry in 60.789127ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:55:10.367669918Z","hostname":"buskin","pid":1555,"instance_id":"057c7948-d027-4c43-bcaf-62810b07fef3","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_a11cf978-990f-48d4-8597-cbd622e7cdbe\") failed: Property not found. retry in 60.826702ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:55:10.43891433Z","hostname":"buskin","pid":1555,"instance_id":"057c7948-d027-4c43-bcaf-62810b07fef3","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_a11cf978-990f-48d4-8597-cbd622e7cdbe\") failed: Property not found. retry in 171.640499ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:55:10.509157218Z","hostname":"buskin","pid":1555,"instance_id":"057c7948-d027-4c43-bcaf-62810b07fef3","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_a11cf978-990f-48d4-8597-cbd622e7cdbe\") failed: Property not found. retry in 226.970543ms","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:55:10.690953584Z","hostname":"buskin","pid":1555,"instance_id":"057c7948-d027-4c43-bcaf-62810b07fef3","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Propolis SMF service is online","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:10.929012188Z","hostname":"buskin","pid":1555,"instance_id":"057c7948-d027-4c43-bcaf-62810b07fef3","component":"InstanceManager","file":"sled-agent/src/instance.rs:1469"} {"msg":"Propolis HTTP server online","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:10.965523944Z","hostname":"buskin","pid":1555,"instance_id":"057c7948-d027-4c43-bcaf-62810b07fef3","component":"InstanceManager","file":"sled-agent/src/instance.rs:1483"} {"msg":"Sending ensure request to propolis: InstanceEnsureRequest { cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAACL4S75WPlYAADhLvlYAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAjOEu+Vj5WAAA4S75WAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiIwNTdjNzk0OC1kMDI3LTRjNDMtYmNhZi02MjgxMGIwN2ZlZjMiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJUDhENWJISjJLSUVxa2VTVm1YNUZ6Z21oMGVlbFU3MHpOY0p6bU1rbkgwNSJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\"), disks: [DiskRequest { device: \"nvme\", name: \"disk-e05b56d916a\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 4b57adae-59ff-4471-a606-c565e10339d7, read_only_parent: Some(Volume { block_size: 512, id: d266589e-cb07-4700-8896-7ca6fc087b4e, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 2a120dbc-0998-48d4-bfb5-e87686f7cf0c, key: Some(\"tAz9C6olO0l7NhRkBocuHL03NyM/pboGlS5gY9ANX2s=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::19]:19001\", \"[fd00:1122:3344:101::15]:19002\", \"[fd00:1122:3344:101::17]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 4b57adae-59ff-4471-a606-c565e10339d7, key: Some(\"EYyu4JaQfaGj+EquIBBl+na9yNkcVzIJU1YGUxTiCks=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::17]:19002\", \"[fd00:1122:3344:101::1a]:19001\", \"[fd00:1122:3344:101::13]:19000\"] } }] } }], migrate: None, nics: [NetworkInterfaceRequest { name: \"opte6\", slot: Slot(0) }], properties: InstanceProperties { bootrom_id: 00000000-0000-0000-0000-000000000000, description: \"Test description\", id: 057c7948-d027-4c43-bcaf-62810b07fef3, image_id: 00000000-0000-0000-0000-000000000000, memory: 1024, metadata: InstanceMetadata { project_id: 5b8f924c-3552-42d1-99ac-94cb0940fe4b, silo_id: bb76b9e7-26a5-4e9e-a61c-3de62341dc71 }, name: \"localshark\", vcpus: 2 } }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:10.968235369Z","hostname":"buskin","pid":1555,"instance_id":"057c7948-d027-4c43-bcaf-62810b07fef3","component":"InstanceManager","file":"sled-agent/src/instance.rs:698"} {"msg":"result of instance_ensure call is Ok(InstanceEnsureResponse { migrate: None })","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:11.188689114Z","hostname":"buskin","pid":1555,"instance_id":"057c7948-d027-4c43-bcaf-62810b07fef3","component":"InstanceManager","file":"sled-agent/src/instance.rs:700"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Starting), migration_status: NoMigration, time: 2024-07-25T05:55:11.190034546Z }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:11.19063613Z","hostname":"buskin","pid":1555,"instance_id":"057c7948-d027-4c43-bcaf-62810b07fef3","component":"InstanceManager","file":"sled-agent/src/instance.rs:584"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:11.190664601Z","hostname":"buskin","pid":1555,"instance_id":"057c7948-d027-4c43-bcaf-62810b07fef3","component":"InstanceManager","file":"sled-agent/src/instance.rs:606","new_vmm_state":"VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-07-25T05:55:11.190034546Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(a11cf978-990f-48d4-8597-cbd622e7cdbe (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-07-25T05:55:03.025890951Z }","propolis_id":"a11cf978-990f-48d4-8597-cbd622e7cdbe"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:11.190676941Z","hostname":"buskin","pid":1555,"instance_id":"057c7948-d027-4c43-bcaf-62810b07fef3","component":"InstanceManager","file":"sled-agent/src/instance.rs:503","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(a11cf978-990f-48d4-8597-cbd622e7cdbe (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-07-25T05:55:03.025890951Z }, propolis_id: a11cf978-990f-48d4-8597-cbd622e7cdbe (propolis), vmm_state: VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-07-25T05:55:11.190034546Z }, migration_state: None }","instance_id":"057c7948-d027-4c43-bcaf-62810b07fef3"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Running), migration_status: NoMigration, time: 2024-07-25T05:55:11.286801554Z }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:11.286840115Z","hostname":"buskin","pid":1555,"instance_id":"057c7948-d027-4c43-bcaf-62810b07fef3","component":"InstanceManager","file":"sled-agent/src/instance.rs:584"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:11.286920056Z","hostname":"buskin","pid":1555,"instance_id":"057c7948-d027-4c43-bcaf-62810b07fef3","component":"InstanceManager","file":"sled-agent/src/instance.rs:606","new_vmm_state":"VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-07-25T05:55:11.286801554Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(a11cf978-990f-48d4-8597-cbd622e7cdbe (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-07-25T05:55:03.025890951Z }","propolis_id":"a11cf978-990f-48d4-8597-cbd622e7cdbe"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:11.286943717Z","hostname":"buskin","pid":1555,"instance_id":"057c7948-d027-4c43-bcaf-62810b07fef3","component":"InstanceManager","file":"sled-agent/src/instance.rs:503","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(a11cf978-990f-48d4-8597-cbd622e7cdbe (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-07-25T05:55:03.025890951Z }, propolis_id: a11cf978-990f-48d4-8597-cbd622e7cdbe (propolis), vmm_state: VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-07-25T05:55:11.286801554Z }, migration_state: None }","instance_id":"057c7948-d027-4c43-bcaf-62810b07fef3"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:11.324432407Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:56252"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:11.325157095Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"e964c7fe-a81b-48b0-a5e7-7b3866a7be92","remote_addr":"[fd00:1122:3344:101::c]:56252","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":631,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:11.370418759Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"6ffa47ff-424a-4847-bf1c-62e02894beeb","remote_addr":"[fd00:1122:3344:101::c]:56252","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":249,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:11.408084954Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:63496"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:11.408162316Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"09080e83-f691-489d-8d2c-8e49b8cfd668","remote_addr":"[fd00:1122:3344:101::c]:63496","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":39,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:11.419196578Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"690227e3-c0a7-40cf-889a-5f339e6e728d","remote_addr":"[fd00:1122:3344:101::c]:63496","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":85,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:13.238844927Z","hostname":"buskin","pid":1555,"uri":"/8fa4bc78-1026-41f0-b09d-c67224071376","method":"GET","req_id":"d7ec3b77-0923-4cb1-8212-32659a968203","remote_addr":"[fd00:1122:3344:101::d]:46986","local_addr":"[fd00:1122:3344:101::1]:57429","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","component":"producer-server","component":"MetricsManager","sled_id":"8fa4bc78-1026-41f0-b09d-c67224071376","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":596,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:21.245229826Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:59198"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:21.245933073Z","hostname":"buskin","pid":1555,"uri":"/instances/057c7948-d027-4c43-bcaf-62810b07fef3/state","method":"PUT","req_id":"68a5c469-60a5-4133-a96e-ccabc9aa7524","remote_addr":"[fd00:1122:3344:101::a]:59198","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":661,"response_code":"200"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Stopping), migration_status: NoMigration, time: 2024-07-25T05:55:21.246075796Z }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:21.246101387Z","hostname":"buskin","pid":1555,"instance_id":"057c7948-d027-4c43-bcaf-62810b07fef3","component":"InstanceManager","file":"sled-agent/src/instance.rs:584"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:21.246180128Z","hostname":"buskin","pid":1555,"instance_id":"057c7948-d027-4c43-bcaf-62810b07fef3","component":"InstanceManager","file":"sled-agent/src/instance.rs:606","new_vmm_state":"VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-07-25T05:55:21.246075796Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(a11cf978-990f-48d4-8597-cbd622e7cdbe (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-07-25T05:55:03.025890951Z }","propolis_id":"a11cf978-990f-48d4-8597-cbd622e7cdbe"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:21.246209959Z","hostname":"buskin","pid":1555,"instance_id":"057c7948-d027-4c43-bcaf-62810b07fef3","component":"InstanceManager","file":"sled-agent/src/instance.rs:503","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(a11cf978-990f-48d4-8597-cbd622e7cdbe (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-07-25T05:55:03.025890951Z }, propolis_id: a11cf978-990f-48d4-8597-cbd622e7cdbe (propolis), vmm_state: VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-07-25T05:55:21.246075796Z }, migration_state: None }","instance_id":"057c7948-d027-4c43-bcaf-62810b07fef3"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Destroyed), migration_status: NoMigration, time: 2024-07-25T05:55:21.337558358Z }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:21.337579669Z","hostname":"buskin","pid":1555,"instance_id":"057c7948-d027-4c43-bcaf-62810b07fef3","component":"InstanceManager","file":"sled-agent/src/instance.rs:584"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:21.33760143Z","hostname":"buskin","pid":1555,"instance_id":"057c7948-d027-4c43-bcaf-62810b07fef3","component":"InstanceManager","file":"sled-agent/src/instance.rs:606","new_vmm_state":"VmmRuntimeState { state: Destroyed, gen: Generation(6), time_updated: 2024-07-25T05:55:21.337558358Z }","new_instance_state":"InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-07-25T05:55:21.337558358Z }","propolis_id":"a11cf978-990f-48d4-8597-cbd622e7cdbe"} {"msg":"terminating VMM that has exited","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:21.33762073Z","hostname":"buskin","pid":1555,"instance_id":"057c7948-d027-4c43-bcaf-62810b07fef3","component":"InstanceManager","file":"sled-agent/src/instance.rs:621","instance_id":"057c7948-d027-4c43-bcaf-62810b07fef3"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:21.33763136Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:21.337915867Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:404"} {"msg":"creating zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:21.337988809Z","hostname":"buskin","pid":1555,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:445","context":"ZoneBundleContext { storage_dirs: [\"/pool/int/6d2aa15e-618e-402e-b368-cf27fd8def68/debug/bundle/zone\", \"/pool/int/75c788e1-45ef-4fcf-958d-db07331bef52/debug/bundle/zone\"], cause: TerminatedInstance, extra_log_dirs: [\"/pool/ext/dd1c81eb-5ddf-42bb-acfe-348d2d141fca/crypt/debug\", \"/pool/ext/59d3cc5a-40b7-44b8-b2af-cfc21755b8bf/crypt/debug\", \"/pool/ext/e78b4f40-df20-4bcf-bd8a-b80afa109d70/crypt/debug\", \"/pool/ext/63a12260-6721-4f51-85d5-262aedc90628/crypt/debug\", \"/pool/ext/587239d7-a145-4bb7-b24a-aff014195deb/crypt/debug\", \"/pool/ext/50e63493-6b9c-42de-846b-797a312b5fbc/crypt/debug\", \"/pool/ext/fd75587e-5d72-48df-a44e-8602b2d61469/crypt/debug\", \"/pool/ext/20d995c8-4908-4380-8438-c04a6e3adea2/crypt/debug\", \"/pool/ext/e43689a1-378e-4be8-9661-3bc522647c6d/crypt/debug\"] }","zone_name":"oxz_propolis-server_a11cf978-990f-48d4-8597-cbd622e7cdbe"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:21.375600252Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:64935"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:21.376570175Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"64b7b6bb-8636-40dc-9986-3302fcc88b23","remote_addr":"[fd00:1122:3344:101::c]:64935","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":887,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:21.389300217Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"0e7245a1-f108-41d2-b1eb-e8782e80738d","remote_addr":"[fd00:1122:3344:101::c]:64935","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":181,"response_code":"204"} {"msg":"finished zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:22.129175496Z","hostname":"buskin","pid":1555,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:1246","metadata":"ZoneBundleMetadata { id: ZoneBundleId { zone_name: \"oxz_propolis-server_a11cf978-990f-48d4-8597-cbd622e7cdbe\", bundle_id: 295d4e51-3915-42ac-973a-c4827e4b067e }, time_created: 2024-07-25T05:55:21.338268985Z, version: 0, cause: TerminatedInstance }"} {"msg":"Halting and removing zone: oxz_propolis-server_a11cf978-990f-48d4-8597-cbd622e7cdbe","v":0,"name":"SledAgent","level":40,"time":"2024-07-25T05:55:22.129285049Z","hostname":"buskin","pid":1555,"instance_id":"057c7948-d027-4c43-bcaf-62810b07fef3","component":"InstanceManager","file":"sled-agent/src/instance.rs:793"} {"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:23.912366049Z","hostname":"buskin","pid":1555,"instance_id":"057c7948-d027-4c43-bcaf-62810b07fef3","component":"InstanceManager","file":"illumos-utils/src/zone.rs:270"} {"msg":"Removed route set for subnet","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:23.928590825Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:906","id":"RouterId { vni: Vni(9219180), kind: System }"} {"msg":"Removed route set for subnet","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:23.928656816Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:906","id":"RouterId { vni: Vni(9219180), kind: Custom(V4(Ipv4Net { addr: 172.30.0.0, width: 22 })) }"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:23.929176269Z","hostname":"buskin","pid":1555,"instance_id":"057c7948-d027-4c43-bcaf-62810b07fef3","component":"InstanceManager","file":"sled-agent/src/instance.rs:503","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-07-25T05:55:21.337558358Z }, propolis_id: a11cf978-990f-48d4-8597-cbd622e7cdbe (propolis), vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-07-25T05:55:23.929142718Z }, migration_state: None }","instance_id":"057c7948-d027-4c43-bcaf-62810b07fef3"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:24.01049928Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:41256"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:24.010768736Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"38051a53-22d9-4872-a0c4-eefc4c28fce8","remote_addr":"[fd00:1122:3344:101::c]:41256","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":219,"response_code":"200"} {"msg":"Mapping virtual NIC to physical host","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:24.011044513Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:780","mapping":"VirtualNetworkInterfaceHost { virtual_ip: 172.30.0.5, virtual_mac: MacAddr(MacAddr6([168, 64, 37, 243, 50, 113])), physical_host_ip: fd00:1122:3344:101::1, vni: Vni(9219180) }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:24.011089904Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"PUT","req_id":"5b62786b-f36b-46c5-a35f-9ba45e263c30","remote_addr":"[fd00:1122:3344:101::c]:41256","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":129,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:24.512779647Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:55738"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:24.520596573Z","hostname":"buskin","pid":1555,"uri":"/instances/057c7948-d027-4c43-bcaf-62810b07fef3/state","method":"GET","req_id":"c5d8cf51-cca1-4f39-bfec-c8f88eecbb87","remote_addr":"[fd00:1122:3344:101::a]:55738","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:902","error_message_external":"Not Found","error_message_internal":"No such instance ID: 057c7948-d027-4c43-bcaf-62810b07fef3","latency_us":911,"response_code":"404"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:24.740712459Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:64972"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:24.740827312Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"e894df83-4e8e-4b94-9347-7432bbdc8fba","remote_addr":"[fd00:1122:3344:101::a]:64972","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":73,"response_code":"200"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:24.746416204Z","hostname":"buskin","pid":1555,"instance_id":"057c7948-d027-4c43-bcaf-62810b07fef3","component":"InstanceManager","file":"sled-agent/src/instance.rs:503","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-07-25T05:55:21.337558358Z }, propolis_id: a11cf978-990f-48d4-8597-cbd622e7cdbe (propolis), vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-07-25T05:55:23.929142718Z }, migration_state: None }","instance_id":"057c7948-d027-4c43-bcaf-62810b07fef3"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:24.747725495Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"dbc39164-9c2e-49df-83d8-8c884ab2d3f5","remote_addr":"[fd00:1122:3344:101::a]:64972","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":201,"response_code":"204"} {"msg":"State monitoring task complete","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:24.750310457Z","hostname":"buskin","pid":1555,"instance_id":"057c7948-d027-4c43-bcaf-62810b07fef3","component":"InstanceManager","file":"sled-agent/src/instance.rs:736"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:24.774117742Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:39154"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:24.774282526Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"c7daba7f-c4fc-4149-8c46-2b003ddd5a1a","remote_addr":"[fd00:1122:3344:101::a]:39154","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":138,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:24.964735019Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:60946"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:24.964827191Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"0361620a-9789-49d0-a517-5b226feaf1ec","remote_addr":"[fd00:1122:3344:101::b]:60946","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":60,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:24.971706844Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"713115ff-39ac-4cb8-a836-143958c84fbc","remote_addr":"[fd00:1122:3344:101::b]:60946","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":94,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:25.021371294Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:53414"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:25.021515587Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"8e9c8cdc-d61f-414e-8abb-ffa8f98e5323","remote_addr":"[fd00:1122:3344:101::b]:53414","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":102,"response_code":"200"} {"msg":"Clearing mapping of virtual NIC to physical host","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:25.027797626Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:820","mapping":"VirtualNetworkInterfaceHost { virtual_ip: 172.30.0.5, virtual_mac: MacAddr(MacAddr6([168, 64, 37, 243, 50, 113])), physical_host_ip: fd00:1122:3344:101::1, vni: Vni(9219180) }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:25.027846057Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"DELETE","req_id":"dbc6d02a-baee-4a2a-92b0-554786c2f403","remote_addr":"[fd00:1122:3344:101::b]:53414","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":6147,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:25.063414112Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:49626"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:25.063473753Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3824"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:25.093986908Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3835"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:25.118825978Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"403e50ca-1d1b-4e93-8615-6f9e5a8aca60","remote_addr":"[fd00:1122:3344:101::a]:49626","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":55372,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:25.119023523Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:58369"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:25.119116755Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:58381"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:25.119176326Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"1e4fed08-c16c-4a19-b020-ad2e669545b6","remote_addr":"[fd00:1122:3344:101::c]:58369","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":102,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:25.119370761Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"8962ddb6-5be9-4c5a-9c0d-5bea2f1a9185","remote_addr":"[fd00:1122:3344:101::a]:49626","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":223,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:25.119469543Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"40a41601-ec37-43c9-a802-e9ef0b84ca5a","remote_addr":"[fd00:1122:3344:101::c]:58381","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":245,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:25.127214397Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"f38e0d2e-13b4-4d43-abb7-4a8829828050","remote_addr":"[fd00:1122:3344:101::c]:58369","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":825,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:25.248729723Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:37824"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:25.248790944Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3824"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:25.275401156Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3835"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:25.299425887Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"cd17efcb-1c77-4efb-a03f-7a3c87291fc3","remote_addr":"[fd00:1122:3344:101::b]:37824","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":50655,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:25.299551839Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:59740"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:25.299641152Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"4563e512-52a7-40c6-ae4f-a807bf0ed37c","remote_addr":"[fd00:1122:3344:101::c]:59740","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":58,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:25.299786965Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"0c292f91-818c-49ee-bfc0-6ecc9bbd020f","remote_addr":"[fd00:1122:3344:101::b]:37824","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":84,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:25.306917394Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"9e1288d6-fb6c-41cf-9afb-272ec581ba0e","remote_addr":"[fd00:1122:3344:101::c]:59740","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":127,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:25.325602398Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:44630"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:25.32569011Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3824"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:25.349369502Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3835"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:25.372641725Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"594ce8f2-d127-4bb8-b228-54ee3cf34733","remote_addr":"[fd00:1122:3344:101::c]:44630","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":47000,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:25.372886231Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"e5276950-241b-42cd-8e3f-6d27dcfbf82c","remote_addr":"[fd00:1122:3344:101::c]:44630","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":86,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:25.533062364Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:63648"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:25.533665429Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"e0abcac7-d022-4424-ba44-1b3f2a33c1ca","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:63648","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":556,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:25.723740482Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:33615"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:25.723932486Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"d71bdeb8-e01a-411a-b1c5-470d76ad7328","remote_addr":"[fd00:1122:3344:101::a]:33615","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":170,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:27.666837472Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:62550"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:27.667503138Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"72ba84e9-1530-4813-8891-25bfa1efbefc","remote_addr":"[fd00:1122:3344:101::a]:62550","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":638,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-07-25T05:55:27.6730293Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"2aefdf84-30fd-433c-a798-94507af00945","remote_addr":"[fd00:1122:3344:101::a]:62550","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/7b594d0/dropshot/src/server.rs:914","latency_us":122,"response_code":"204"}