[ Aug 5 16:42:32 Enabled. ] [ Aug 5 16:42:32 Executing start method ("ctrun -l child -o noorphan,regent /opt/oxide/sled-agent/sled-agent run /opt/oxide/sled-agent/pkg/config.toml &"). ] [ Aug 5 16:42:32 Method "start" exited with status 0. ] [ Aug 5 16:42:32 Rereading configuration. ] [ Aug 5 16:42:32 No 'refresh' method defined. Treating as :true. ] note: configured to log to "/dev/stdout" {"msg":"Ensuring contract reaper thread","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:32.209539814Z","hostname":"buskin","pid":1555,"file":"illumos-utils/src/running_zone.rs:113"} {"msg":"finding links [\"net0\", \"net1\"]","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:32.209730437Z","hostname":"buskin","pid":1555,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:255"} {"msg":"Starting mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:32.262422916Z","hostname":"buskin","pid":1555,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:262","addr-objs":"[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]"} {"msg":"Importing mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:32.262467316Z","hostname":"buskin","pid":1555,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:42","path":"/opt/oxide/mg-ddm/pkg/ddm/manifest.xml"} {"msg":"Setting mg-ddm interfaces","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:32.376998289Z","hostname":"buskin","pid":1555,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:50","interfaces":"(\"net0/ll\" \"net1/ll\")"} {"msg":"Enabling mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:32.385568885Z","hostname":"buskin","pid":1555,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/maghemite.rs:56"} {"msg":"Ensuring zfs key directory exists","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:32.393329558Z","hostname":"buskin","pid":1555,"component":"BootstrapAgentStartup","file":"sled-agent/src/bootstrap/pre_server.rs:272","path":"/var/run/oxide/"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:32.991984297Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fdb0:18c0:4d0d:9fb2::, width: 64 }"} {"msg":"Starting KeyManager","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.132448512Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:125"} {"msg":"Starting StorageManager","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.132481833Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:137"} {"msg":"Starting StorageMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.132598885Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:150"} {"msg":"DumpSetup poll loop started.","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.132656626Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:520"} {"msg":"Starting HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.132666986Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:172","nongimlet_observed_disks":"[]","sled_mode":"Scrimlet { asic: SoftNpuZone }"} {"msg":"Creating HardwareManager","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.132682216Z","hostname":"buskin","pid":1555,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:716"} {"msg":"Generated i86pc baseboard Pc { identifier: \"buskin\", model: \"i86pc\" }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.135778719Z","hostname":"buskin","pid":1555,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:623"} {"msg":"Device is not a Gimlet (i86pc), proceeding with null hardware view","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:42:33.135785549Z","hostname":"buskin","pid":1555,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:765"} {"msg":"Starting HardwareMonitor","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.135801959Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:186"} {"msg":"Monitoring for hardware updates","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.135807499Z","hostname":"buskin","pid":1555,"component":"HardwareManager","file":"sled-hardware/src/illumos/mod.rs:824"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.1358323Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:242","vdev":"/scratch/m2_0.vdev"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.135871791Z","hostname":"buskin","pid":1555,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:271","disks":"{}","underlay_network_info":"None"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.136016923Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.136047994Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.136058324Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/m2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.147030381Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.147054411Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:371","id":"4576fedf-479b-43b3-8fad-bb8b7ec274f8 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.266441277Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:304","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: 4576fedf-479b-43b3-8fad-bb8b7ec274f8 (zpool), kind: Internal }"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.50289857Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.50293248Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:242","vdev":"/scratch/m2_1.vdev"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.50293785Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:601","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.50294856Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:622","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.502956331Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.502974631Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.502979321Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/m2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/m2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.511883493Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"} {"msg":"Formatting zpool with generated ID","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.511904543Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:371","id":"d09b3f64-0a36-4052-8233-a94e62e662e2 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.620289082Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:304","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: d09b3f64-0a36-4052-8233-a94e62e662e2 (zpool), kind: Internal }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.858562235Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:242","vdev":"/scratch/u2_0.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.858603846Z","hostname":"buskin","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 4576fedf-479b-43b3-8fad-bb8b7ec274f8 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: d09b3f64-0a36-4052-8233-a94e62e662e2 (zpool), kind: Internal } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.858636566Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.858645737Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.858653157Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.858667027Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.858711427Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:242","vdev":"/scratch/u2_1.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.858726528Z","hostname":"buskin","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 4576fedf-479b-43b3-8fad-bb8b7ec274f8 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: d09b3f64-0a36-4052-8233-a94e62e662e2 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.858742238Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.858747138Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.858751788Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.858766579Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.858769999Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:242","vdev":"/scratch/u2_2.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.858775069Z","hostname":"buskin","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 4576fedf-479b-43b3-8fad-bb8b7ec274f8 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: d09b3f64-0a36-4052-8233-a94e62e662e2 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.858789849Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.858795389Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.858799249Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.858803719Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.858807589Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:242","vdev":"/scratch/u2_3.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.858815519Z","hostname":"buskin","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 4576fedf-479b-43b3-8fad-bb8b7ec274f8 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: d09b3f64-0a36-4052-8233-a94e62e662e2 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.85883073Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.85884409Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.85884781Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.85885181Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.85885631Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:242","vdev":"/scratch/u2_4.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.85885987Z","hostname":"buskin","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 4576fedf-479b-43b3-8fad-bb8b7ec274f8 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: d09b3f64-0a36-4052-8233-a94e62e662e2 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.85887646Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.858880621Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.85888517Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.858889401Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.858894061Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:242","vdev":"/scratch/u2_5.vdev"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.858897801Z","hostname":"buskin","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 4576fedf-479b-43b3-8fad-bb8b7ec274f8 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: d09b3f64-0a36-4052-8233-a94e62e662e2 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.858921461Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.858925691Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.858930221Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.858937071Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.858973632Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:242","vdev":"/scratch/u2_6.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.858988572Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.858993882Z","hostname":"buskin","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 4576fedf-479b-43b3-8fad-bb8b7ec274f8 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: d09b3f64-0a36-4052-8233-a94e62e662e2 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.859028153Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.859032873Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.859037993Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.859041383Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:242","vdev":"/scratch/u2_7.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.859050583Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.859055313Z","hostname":"buskin","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 4576fedf-479b-43b3-8fad-bb8b7ec274f8 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: d09b3f64-0a36-4052-8233-a94e62e662e2 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.859076364Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.859092874Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.859097614Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Upserting synthetic device to Storage Manager","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.859101634Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:242","vdev":"/scratch/u2_8.vdev"} {"msg":"Received DetectedRawDisk { raw_disk: Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }), tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.859106414Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.859110694Z","hostname":"buskin","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 4576fedf-479b-43b3-8fad-bb8b7ec274f8 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: d09b3f64-0a36-4052-8233-a94e62e662e2 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Inserting disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.859138355Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:529","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.859142765Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.859147195Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Waiting for boot disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.859150685Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:96"} {"msg":"Found boot disk DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.859160265Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:98"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.859166685Z","hostname":"buskin","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 4576fedf-479b-43b3-8fad-bb8b7ec274f8 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: d09b3f64-0a36-4052-8233-a94e62e662e2 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: Unmanaged(Synthetic(RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.859193176Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.859199116Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.859202526Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Starting Bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.859206216Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:210"} {"msg":"No ledger in /pool/int/4576fedf-479b-43b3-8fad-bb8b7ec274f8/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.859268827Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/d09b3f64-0a36-4052-8233-a94e62e662e2/cluster/bootstore-fsm-state.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.859285057Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found. Loading Fsm::State as Uninitialized","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.859293568Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:83"} {"msg":"No ledger in /pool/int/4576fedf-479b-43b3-8fad-bb8b7ec274f8/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.859308768Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/d09b3f64-0a36-4052-8233-a94e62e662e2/cluster/bootstore-network-config.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.859313318Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:188"} {"msg":"No ledger found for network config","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.859322158Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/storage.rs:146"} {"msg":"Starting Bootstore DDMD poller","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.859347478Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:215"} {"msg":"Starting ZoneBundler related tasks","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.859351918Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/long_running_tasks.rs:230"} {"msg":"Creating ServiceManager","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.870406247Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:725"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.870440548Z","hostname":"buskin","pid":1555,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:292","time_to_next_cleanup":"599.999915219s","next_cleanup":"Instant { tv_sec: 11490, tv_nsec: 811279875 }"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.870511259Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3664"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.870530659Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"No ledger in /pool/int/4576fedf-479b-43b3-8fad-bb8b7ec274f8/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.87060463Z","hostname":"buskin","pid":1555,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/d09b3f64-0a36-4052-8233-a94e62e662e2/config/sled-agent-request.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.872893489Z","hostname":"buskin","pid":1555,"component":"BootstrapAgentStartup","file":"common/src/ledger.rs:188"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.873309696Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:205"} {"msg":"Started listening","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.873382058Z","hostname":"buskin","pid":1555,"component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:44","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Injecting RSS configuration: RackInitializeRequest {\n trust_quorum_peers: None,\n bootstrap_discovery: OnlyOurs,\n ntp_servers: [\n \"0.pool.ntp.org\",\n ],\n dns_servers: [\n 1.1.1.1,\n 9.9.9.9,\n ],\n internal_services_ip_pool_ranges: [\n V4(\n Ipv4Range {\n first: 10.151.1.164,\n last: 10.151.1.173,\n },\n ),\n ],\n external_dns_ips: [\n 10.151.1.164,\n 10.151.1.165,\n ],\n external_dns_zone_name: \"oxide.test\",\n external_certificates: \"\",\n recovery_silo: RecoverySiloConfig {\n silo_name: Name(\n \"recovery\",\n ),\n user_name: UserId(\n \"recovery\",\n ),\n user_password_hash: NewPasswordHash(\n PasswordHashString {\n string: \"$argon2id$v=19$m=98304,t=13,p=1$RUlWc0ZxaHo0WFdrN0N6ZQ$S8p52j85GPvMhR/ek3GL0el/oProgTwWpHJZ8lsQQoY\",\n encoding: B64,\n },\n ),\n },\n rack_network_config: RackNetworkConfigV2 {\n rack_subnet: Ipv6Net {\n addr: fd00:1122:3344:100::,\n width: 56,\n },\n infra_ip_first: 10.151.1.174,\n infra_ip_last: 10.151.1.174,\n ports: [\n PortConfigV2 {\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n nexthop: 10.151.1.100,\n vlan_id: None,\n },\n ],\n addresses: [\n UplinkAddressConfig {\n address: V4(\n Ipv4Net {\n addr: 10.151.1.174,\n width: 24,\n },\n ),\n vlan_id: None,\n },\n ],\n switch: Switch0,\n port: \"qsfp0\",\n uplink_port_speed: Speed40G,\n uplink_port_fec: None,\n bgp_peers: [],\n autoneg: false,\n },\n ],\n bgp: [],\n bfd: [],\n },\n allowed_source_ips: Any,\n}","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.873610922Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1017"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.873618552Z","hostname":"buskin","pid":1555,"component":"DnsResolver","component":"RSS","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.873661342Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"No ledger in /pool/int/4576fedf-479b-43b3-8fad-bb8b7ec274f8/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.873702573Z","hostname":"buskin","pid":1555,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/d09b3f64-0a36-4052-8233-a94e62e662e2/config/rss-plan-completed.marker","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.873713623Z","hostname":"buskin","pid":1555,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"RSS configuration has not been fully applied yet","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.873721234Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1077"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.873744394Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"No ledger in /pool/int/4576fedf-479b-43b3-8fad-bb8b7ec274f8/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.873752584Z","hostname":"buskin","pid":1555,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/d09b3f64-0a36-4052-8233-a94e62e662e2/config/rss-sled-plan.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.873762624Z","hostname":"buskin","pid":1555,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"Creating new allocation plan","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.873767544Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1118"} {"msg":"Serializing plan","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.873771344Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:156"} {"msg":"Creating plan for the sled at fdb0:18c0:4d0d:9fb2::1","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.873774904Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:129"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.873789745Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Writing ledger to /pool/int/4576fedf-479b-43b3-8fad-bb8b7ec274f8/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.873810855Z","hostname":"buskin","pid":1555,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/d09b3f64-0a36-4052-8233-a94e62e662e2/config/.rss-sled-plan.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.874136871Z","hostname":"buskin","pid":1555,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Sled plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.874315234Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/plan/sled.rs:176"} {"msg":"Writing Rack Network Configuration to bootstore","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.874320304Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1151"} {"msg":"Attempting to update network config with generation: 1, current_generation: 0","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.874380695Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/4576fedf-479b-43b3-8fad-bb8b7ec274f8/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.874386155Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/d09b3f64-0a36-4052-8233-a94e62e662e2/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.874554398Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 1","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.874720931Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"Received initialization request from RSS","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.874841053Z","hostname":"buskin","pid":1555,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:217","target_sled":"[fdb0:18c0:4d0d:9fb2::1]:12346","request":"StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: 3abf3a4f-8820-40af-a01f-4369b42a5529, rack_id: b682b2a4-3c99-42b2-9e14-9bba53281f21, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }"} {"msg":"Accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.87528817Z","hostname":"buskin","pid":1555,"remote_addr":"[fdb0:18c0:4d0d:9fb2::1]:58920","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:66"} {"msg":"Loading Sled Agent: StartSledAgentRequest { generation: 0, schema_version: 1, body: StartSledAgentRequestBody { id: 3abf3a4f-8820-40af-a01f-4369b42a5529, rack_id: b682b2a4-3c99-42b2-9e14-9bba53281f21, use_trust_quorum: false, is_lrtq_learner: false, subnet: Ipv6Subnet { net: Ipv6Net { addr: fd00:1122:3344:101::, width: 64 } } } }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.875387872Z","hostname":"buskin","pid":1555,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:359"} {"msg":"KeyManager: using hardcoded secret retriever","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.875396102Z","hostname":"buskin","pid":1555,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:374"} {"msg":"setting up sled agent server","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.875432863Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/server.rs:44"} {"msg":"Received KeyManagerReady","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.875450523Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Loading storage config","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.875456833Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:573"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.875460293Z","hostname":"buskin","pid":1555,"component":"DnsResolver","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"No ledger in /pool/int/4576fedf-479b-43b3-8fad-bb8b7ec274f8/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.875465193Z","hostname":"buskin","pid":1555,"request":"load_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.875472224Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:101::, width: 64 }"} {"msg":"No ledger in /pool/int/d09b3f64-0a36-4052-8233-a94e62e662e2/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.875478033Z","hostname":"buskin","pid":1555,"request":"load_ledger","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger of physical disks exists","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.875486794Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:562"} {"msg":"KeyManager ready, but no ledger detected","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.875492124Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:590"} {"msg":"automatically managing disks","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.875495724Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:394"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.875501054Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:374"} {"msg":"Successfully synchronized disks without error","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.875504804Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:480","result":"DisksManagementResult { status: [] }"} {"msg":"Enabling switch zone (new)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.877449327Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3898"} {"msg":"Starting switch zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.877495558Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:4271"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.907311066Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Configuring new Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.928676571Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_switch","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.946897911Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"SledAgent::new(..) starting","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.987413872Z","hostname":"buskin","pid":1555,"sled_id":"3abf3a4f-8820-40af-a01f-4369b42a5529","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:383"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.987473653Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Requested swap device of size 64 GiB","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.987479703Z","hostname":"buskin","pid":1555,"sled_id":"3abf3a4f-8820-40af-a01f-4369b42a5529","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:395"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.987484413Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 59.979987ms)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:33.9908497Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"008a4174-5d87-469f-988e-cf2880a3cae7\", \"content-length\": \"124\", \"date\": \"Mon, 05 Aug 2024 16:42:32 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"008a4174-5d87-469f-988e-cf2880a3cae7\" }"} {"msg":"attempting to create encrypted zvol: name=\"oxi_4576fedf-479b-43b3-8fad-bb8b7ec274f8/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:34.000916492Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/swap_device.rs:153"} {"msg":"successfully created encrypted zvol: name=\"oxi_4576fedf-479b-43b3-8fad-bb8b7ec274f8/swap\", size_gb=64","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:34.028761467Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/swap_device.rs:249"} {"msg":"adding swap device: swapname=\"/dev/zvol/dsk/oxi_4576fedf-479b-43b3-8fad-bb8b7ec274f8/swap\"","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:34.028810318Z","hostname":"buskin","pid":1555,"file":"sled-agent/src/swap_device.rs:96"} {"msg":"Mounting backing filesystems","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:34.034047997Z","hostname":"buskin","pid":1555,"sled_id":"3abf3a4f-8820-40af-a01f-4369b42a5529","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:410"} {"msg":"Processing fmd","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:34.034061067Z","hostname":"buskin","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:126"} {"msg":"Ensuring dataset oxi_4576fedf-479b-43b3-8fad-bb8b7ec274f8/backing/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:34.034070337Z","hostname":"buskin","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:136"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:34.052125065Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 117.429553ms)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:34.052774536Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"8c8efff4-fb87-4159-b342-5e1e6fac9a11\", \"content-length\": \"124\", \"date\": \"Mon, 05 Aug 2024 16:42:34 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"8c8efff4-fb87-4159-b342-5e1e6fac9a11\" }"} {"msg":"Stopping service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:34.125528507Z","hostname":"buskin","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:170"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:34.171081074Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 158.150594ms)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:34.171689664Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"c80d7a6c-e81f-4ed7-9599-a3531481ffa4\", \"content-length\": \"124\", \"date\": \"Mon, 05 Aug 2024 16:42:34 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"c80d7a6c-e81f-4ed7-9599-a3531481ffa4\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:34.331166294Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 233.055129ms)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:34.331770774Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"14002942-0f67-408b-a7c1-b228d309951a\", \"content-length\": \"124\", \"date\": \"Mon, 05 Aug 2024 16:42:34 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"14002942-0f67-408b-a7c1-b228d309951a\" }"} {"msg":"Mounting oxi_4576fedf-479b-43b3-8fad-bb8b7ec274f8/backing/fmd on /var/fm/fmd","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:34.344137655Z","hostname":"buskin","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:178"} {"msg":"Ensuring directory /var/fm/fmd/rsrc","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:34.356249982Z","hostname":"buskin","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Ensuring directory /var/fm/fmd/ckpt","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:34.356355633Z","hostname":"buskin","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Ensuring directory /var/fm/fmd/xprt","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:34.356397714Z","hostname":"buskin","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:186"} {"msg":"Starting service svc:/system/fmd:default","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:34.356428425Z","hostname":"buskin","pid":1555,"component":"BackingFs","file":"sled-agent/src/backing_fs.rs:192"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:34.56723932Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 114.925684ms)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:34.568503331Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f3c4d3d9-d8cb-44d1-b03c-96e11f37401b\", \"content-length\": \"124\", \"date\": \"Mon, 05 Aug 2024 16:42:34 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f3c4d3d9-d8cb-44d1-b03c-96e11f37401b\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:34.685259432Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 325.595018ms)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:34.685889653Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"1808b149-2a7b-44b6-9640-7b8d9a9d9917\", \"content-length\": \"124\", \"date\": \"Mon, 05 Aug 2024 16:42:34 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"1808b149-2a7b-44b6-9640-7b8d9a9d9917\" }"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:35.013031241Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"Failed enable ddm stats (will retry after 255.481837ms)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:35.013743994Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:177","err":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ca680fc1-143e-451a-ab72-89ae42791310\", \"content-length\": \"124\", \"date\": \"Mon, 05 Aug 2024 16:42:34 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"ca680fc1-143e-451a-ab72-89ae42791310\" }"} {"msg":"using '[AddrObject { interface: \"net0\", name: \"ll\" }, AddrObject { interface: \"net1\", name: \"ll\" }]' as data links for xde driver","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:35.090377241Z","hostname":"buskin","pid":1555,"sled_id":"3abf3a4f-8820-40af-a01f-4369b42a5529","component":"SledAgent","file":"illumos-utils/src/opte/illumos.rs:90"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:35.09095968Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:40509","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:205","component":"producer-server","component":"MetricsManager","sled_id":"3abf3a4f-8820-40af-a01f-4369b42a5529","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:205"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:35.090981581Z","hostname":"buskin","pid":1555,"component":"internal-dns-resolver","file":"internal-dns/src/resolver.rs:60","component":"producer-server","component":"MetricsManager","sled_id":"3abf3a4f-8820-40af-a01f-4369b42a5529","component":"SledAgent","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"starting oximeter metric producer server","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:35.091046392Z","hostname":"buskin","pid":1555,"file":"oximeter/producer/src/lib.rs:283","component":"producer-server","component":"MetricsManager","sled_id":"3abf3a4f-8820-40af-a01f-4369b42a5529","component":"SledAgent","file":"oximeter/producer/src/lib.rs:283","interval":"30s","address":"[fd00:1122:3344:101::1]:40509","producer_id":"3abf3a4f-8820-40af-a01f-4369b42a5529"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:42:35.097456031Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","component":"MetricsManager","sled_id":"3abf3a4f-8820-40af-a01f-4369b42a5529","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"139.523994ms"} {"msg":"50% of 102952026112 physical ram = 51476013056 bytes)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:35.139529329Z","hostname":"buskin","pid":1555,"component":"VmmReservoirManager","sled_id":"3abf3a4f-8820-40af-a01f-4369b42a5529","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:267"} {"msg":"Setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:35.139559599Z","hostname":"buskin","pid":1555,"component":"VmmReservoirManager","sled_id":"3abf3a4f-8820-40af-a01f-4369b42a5529","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:275"} {"msg":"sled agent started","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:35.13960855Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:908","underlay_address":"fd00:1122:3344:101::1"} {"msg":"new DNS resolver","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:35.13962081Z","hostname":"buskin","pid":1555,"component":"DnsResolver","component":"ServiceManager","file":"internal-dns/src/resolver.rs:60","addresses":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]"} {"msg":"listening","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:35.140430854Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:205"} {"msg":"Sled Agent loaded; recording configuration","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:35.140455675Z","hostname":"buskin","pid":1555,"component":"SledAgentMain","file":"sled-agent/src/bootstrap/server.rs:429"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:35.140487735Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Writing ledger to /pool/int/4576fedf-479b-43b3-8fad-bb8b7ec274f8/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:35.140512156Z","hostname":"buskin","pid":1555,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/d09b3f64-0a36-4052-8233-a94e62e662e2/config/.sled-agent-request.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:35.14078769Z","hostname":"buskin","pid":1555,"component":"SledAgentMain","file":"common/src/ledger.rs:199"} {"msg":"Sled Agent Started","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:35.140963973Z","hostname":"buskin","pid":1555,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:138"} {"msg":"Checking current full hardware snapshot","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:35.140997234Z","hostname":"buskin","pid":1555,"component":"HardwareMonitor","file":"sled-agent/src/hardware_monitor.rs:271","disks":"{}","underlay_network_info":"Some((fd00:1122:3344:101::2, Some(RackNetworkConfigV2 { rack_subnet: Ipv6Net { addr: fd00:1122:3344:100::, width: 56 }, infra_ip_first: 10.151.1.174, infra_ip_last: 10.151.1.174, ports: [PortConfigV2 { routes: [RouteConfig { destination: V4(Ipv4Net { addr: 0.0.0.0, width: 0 }), nexthop: 10.151.1.100, vlan_id: None }], addresses: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }], switch: Switch0, port: \"qsfp0\", uplink_port_speed: Speed40G, uplink_port_fec: None, bgp_peers: [], autoneg: false }], bgp: [], bfd: [] })))"} {"msg":"Ensuring scrimlet services (enabling services)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:35.141017494Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3664"} {"msg":"Connection closed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:35.141024734Z","hostname":"buskin","pid":1555,"remote_addr":"[fdb0:18c0:4d0d:9fb2::1]:58920","component":"SledAgentSprocketsServer","file":"sled-agent/src/bootstrap/sprockets_server.rs:73"} {"msg":"Peer agent initialized","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:35.141084555Z","hostname":"buskin","pid":1555,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:108","peer":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Initialized sled agent","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:35.141093155Z","hostname":"buskin","pid":1555,"component":"BootstrapAgentRssHandler","file":"sled-agent/src/bootstrap/rss_handle.rs:232","target_sled":"[fdb0:18c0:4d0d:9fb2::1]:12346"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:35.317237689Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"No ledger in /pool/int/4576fedf-479b-43b3-8fad-bb8b7ec274f8/config/rss-service-plan-v3.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:35.31729012Z","hostname":"buskin","pid":1555,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/d09b3f64-0a36-4052-8233-a94e62e662e2/config/rss-service-plan-v3.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:35.317312091Z","hostname":"buskin","pid":1555,"component":"RSS","file":"common/src/ledger.rs:188"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:35.317345711Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:35.317356881Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Enabling ddm stats","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:35.317508604Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:163","dns_servers":"[[fd00:1122:3344:1::1]:53, [fd00:1122:3344:2::1]:53, [fd00:1122:3344:3::1]:53, [fd00:1122:3344:4::1]:53, [fd00:1122:3344:5::1]:53]","addr":"fd00:1122:3344:101::1"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:42:35.318698574Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","component":"MetricsManager","sled_id":"3abf3a4f-8820-40af-a01f-4369b42a5529","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"670.313838ms"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/3abf3a4f-8820-40af-a01f-4369b42a5529): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:42:35.322285505Z","hostname":"buskin","pid":1555,"component":"NexusNotifierTask","sled_id":"3abf3a4f-8820-40af-a01f-4369b42a5529","component":"SledAgent","file":"sled-agent/src/nexus.rs:601"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:35.356778724Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:52434"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:35.356949427Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:35.357033348Z","hostname":"buskin","pid":1555,"uri":"/inventory","method":"GET","req_id":"a1b37d09-e5a2-43a2-93f2-f39db13445d4","remote_addr":"[fd00:1122:3344:101::1]:52434","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":168,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:35.386363888Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:57302"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:35.38647221Z","hostname":"buskin","pid":1555,"uri":"/sled-role","method":"GET","req_id":"3d99c969-1953-438a-92f8-5a34d4319992","remote_addr":"[fd00:1122:3344:101::1]:57302","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":71,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:35.387145702Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Writing ledger to /pool/int/4576fedf-479b-43b3-8fad-bb8b7ec274f8/config/.rss-service-plan-v3.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:35.387168772Z","hostname":"buskin","pid":1555,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/d09b3f64-0a36-4052-8233-a94e62e662e2/config/.rss-service-plan-v3.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:35.387475637Z","hostname":"buskin","pid":1555,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Service plan written to storage","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:35.387728461Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/plan/service.rs:830"} {"msg":"attempting to set up sled's storage: OmicronPhysicalDisksConfig { generation: Generation(1), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, id: a3ae8142-d872-46ed-a8f5-1d52643d5715, pool_id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, id: 5c841a52-6724-4851-93d0-4b1d5629a1af, pool_id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, id: 5fdef7c1-4728-486f-b1af-649d5f2d2aa9, pool_id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, id: 42e8892c-087f-4794-83a0-c0fbf49a30cc, pool_id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, id: 2456e0c2-5c6b-44b7-aebc-71a4d8c49e81, pool_id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, id: ea8136fb-9e0e-47ee-be22-039a7aa90f13, pool_id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, id: 5c671b95-d7f1-40ca-a86f-acf1e606bf26, pool_id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, id: dda42a79-a396-4af3-8e55-625341dd0cd5, pool_id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, id: ec5a428d-5f04-4e94-a324-dd976be4c40e, pool_id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool) }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:35.419142167Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:355"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:35.41929611Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:43904"} {"msg":"physical disks ensure","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:35.419553454Z","hostname":"buskin","pid":1555,"sled_id":"3abf3a4f-8820-40af-a01f-4369b42a5529","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:842"} {"msg":"Received OmicronPhysicalDisksEnsure { config: OmicronPhysicalDisksConfig { generation: Generation(1), disks: [OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, id: a3ae8142-d872-46ed-a8f5-1d52643d5715, pool_id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, id: 5c841a52-6724-4851-93d0-4b1d5629a1af, pool_id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, id: 5fdef7c1-4728-486f-b1af-649d5f2d2aa9, pool_id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, id: 42e8892c-087f-4794-83a0-c0fbf49a30cc, pool_id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, id: 2456e0c2-5c6b-44b7-aebc-71a4d8c49e81, pool_id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, id: ea8136fb-9e0e-47ee-be22-039a7aa90f13, pool_id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, id: 5c671b95-d7f1-40ca-a86f-acf1e606bf26, pool_id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, id: dda42a79-a396-4af3-8e55-625341dd0cd5, pool_id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool) }, OmicronPhysicalDiskConfig { identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, id: ec5a428d-5f04-4e94-a324-dd976be4c40e, pool_id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool) }] }, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:35.419587575Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"No ledger in /pool/int/4576fedf-479b-43b3-8fad-bb8b7ec274f8/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:35.419592945Z","hostname":"buskin","pid":1555,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/d09b3f64-0a36-4052-8233-a94e62e662e2/config/omicron-physical-disks.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:35.419599305Z","hostname":"buskin","pid":1555,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:188"} {"msg":"No previously-stored 'requested disks', creating new ledger","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:35.419607325Z","hostname":"buskin","pid":1555,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"sled-storage/src/manager.rs:644"} {"msg":"Synchronizing disk managment","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:35.419641036Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:374"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:35.419645866Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:414","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:35.419650676Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:491"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:35.419654596Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_0.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_0.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:35.541311671Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:35.541345661Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:366","id":"bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool)"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:42:36.148535806Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","component":"MetricsManager","sled_id":"3abf3a4f-8820-40af-a01f-4369b42a5529","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"668.379393ms"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/3abf3a4f-8820-40af-a01f-4369b42a5529): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-08-05T16:42:36.149056634Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"3abf3a4f-8820-40af-a01f-4369b42a5529","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:36.498040916Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:304","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","zpool":"ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:36.660050519Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:348","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:36.66010931Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:350","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:36.66012074Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:354","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:36.66013616Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:356","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:36.66015212Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:36.660290073Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:36.660297033Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:368"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:42:36.841584114Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","component":"MetricsManager","sled_id":"3abf3a4f-8820-40af-a01f-4369b42a5529","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"2.065833161s"} {"msg":"Setting up dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:37.195476429Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2475"} {"msg":"Setting up lldpd service","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:37.19555833Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2693"} {"msg":"Setting up MGS service","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:37.19558031Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2345"} {"msg":"Setting up wicketd service","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:37.195584811Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2392"} {"msg":"Setting up mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:37.195642642Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2774"} {"msg":"Setting up mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:37.195649802Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2816"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/3abf3a4f-8820-40af-a01f-4369b42a5529): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:42:37.195926707Z","hostname":"buskin","pid":1555,"component":"NexusNotifierTask","sled_id":"3abf3a4f-8820-40af-a01f-4369b42a5529","component":"SledAgent","file":"sled-agent/src/nexus.rs:601"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/3abf3a4f-8820-40af-a01f-4369b42a5529): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-08-05T16:42:37.196248622Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"3abf3a4f-8820-40af-a01f-4369b42a5529","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"Setting up tfport service","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:37.197986731Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2618"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-0-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:37.198091653Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Setting up Simulated SP service","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:37.3808501Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:2386"} {"msg":"Profile for oxz_switch:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:37.381009473Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_switch zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:37.381281477Z","hostname":"buskin","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:472"} {"msg":"Automatically destroying dataset: oxp_bd033efe-47fe-4ba2-8bc4-ba99e628ada7/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:37.539231151Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:411"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 33.73814ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:42:38.131959429Z","hostname":"buskin","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/3abf3a4f-8820-40af-a01f-4369b42a5529): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-08-05T16:42:38.491213415Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"3abf3a4f-8820-40af-a01f-4369b42a5529","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 68.42278ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:42:38.671132694Z","hostname":"buskin","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:42:39.083900902Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","component":"MetricsManager","sled_id":"3abf3a4f-8820-40af-a01f-4369b42a5529","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"4.414958127s"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 147.948151ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:42:39.279939085Z","hostname":"buskin","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/3abf3a4f-8820-40af-a01f-4369b42a5529): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:42:39.631527831Z","hostname":"buskin","pid":1555,"component":"NexusNotifierTask","sled_id":"3abf3a4f-8820-40af-a01f-4369b42a5529","component":"SledAgent","file":"sled-agent/src/nexus.rs:601"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/3abf3a4f-8820-40af-a01f-4369b42a5529): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-08-05T16:42:39.631597843Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"3abf3a4f-8820-40af-a01f-4369b42a5529","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 352.600336ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:42:39.840253351Z","hostname":"buskin","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 815.007348ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:42:40.508586238Z","hostname":"buskin","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Looking for unencrypted datasets in oxp_bd033efe-47fe-4ba2-8bc4-ba99e628ada7","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:40.523259748Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:496"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/3abf3a4f-8820-40af-a01f-4369b42a5529): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-08-05T16:42:40.73040976Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"3abf3a4f-8820-40af-a01f-4369b42a5529","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:40.736100447Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:512","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:40.736115678Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:429","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:40.736129688Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:414","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:40.736136948Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:491"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:40.736141488Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_1.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_1.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:40.955362197Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:40.955386897Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:366","id":"1aa678c2-e32d-4428-a115-7e698777172b (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:42.128470701Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:304","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","zpool":"ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:42.361934493Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:348","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:42.361970523Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:350","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:42.361977304Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:354","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:42.361984613Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:356","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:42.361993944Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:42.362167757Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:42.362180627Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:368"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/3abf3a4f-8820-40af-a01f-4369b42a5529): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-08-05T16:42:43.093940805Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"3abf3a4f-8820-40af-a01f-4369b42a5529","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"wait for service Some(\"oxz_switch\") failed: Property not found. retry in 963.355659ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:42:43.094036137Z","hostname":"buskin","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/3abf3a4f-8820-40af-a01f-4369b42a5529): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:42:43.094122978Z","hostname":"buskin","pid":1555,"component":"NexusNotifierTask","sled_id":"3abf3a4f-8820-40af-a01f-4369b42a5529","component":"SledAgent","file":"sled-agent/src/nexus.rs:601"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-1-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:43.356079976Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"failed to lookup Nexus IP, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:42:43.593530355Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:391","component":"producer-server","component":"MetricsManager","sled_id":"3abf3a4f-8820-40af-a01f-4369b42a5529","component":"SledAgent","file":"oximeter/producer/src/lib.rs:391","error":"\"proto error: io error: No route to host (os error 148)\"","delay":"10.845809831s"} {"msg":"Automatically destroying dataset: oxp_1aa678c2-e32d-4428-a115-7e698777172b/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:43.600328841Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:411"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/3abf3a4f-8820-40af-a01f-4369b42a5529): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-08-05T16:42:44.760647198Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"3abf3a4f-8820-40af-a01f-4369b42a5529","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"Finished setting reservoir size to 49090 MiB bytes","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:45.210496629Z","hostname":"buskin","pid":1555,"component":"VmmReservoirManager","sled_id":"3abf3a4f-8820-40af-a01f-4369b42a5529","component":"SledAgent","file":"sled-agent/src/vmm_reservoir.rs:279"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/3abf3a4f-8820-40af-a01f-4369b42a5529): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:42:45.211819852Z","hostname":"buskin","pid":1555,"component":"NexusNotifierTask","sled_id":"3abf3a4f-8820-40af-a01f-4369b42a5529","component":"SledAgent","file":"sled-agent/src/nexus.rs:601"} {"msg":"Re-enabling running switch zone (new address)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:45.213152874Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3922","new":"[fd00:1122:3344:101::2, ::1]","old":"[::1]"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:45.213160465Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3938"} {"msg":"Adding address: Static(V6(Ipv6Network { addr: fd00:1122:3344:101::2, prefix: 64 }))","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:45.213169485Z","hostname":"buskin","pid":1555,"zone":"oxz_switch","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:518"} {"msg":"Looking for unencrypted datasets in oxp_1aa678c2-e32d-4428-a115-7e698777172b","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:45.31142943Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:496"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:45.319336005Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:512","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:45.319380666Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:429","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:45.319388176Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:414","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:45.319412046Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:491"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:45.319416637Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_2.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_2.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:45.329452008Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:45.329482088Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:366","id":"cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:45.439831Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:304","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","zpool":"ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:45.448426297Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:348","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:45.448464887Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:350","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:45.448473227Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:354","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:45.448494768Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:356","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:45.448506418Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:45.448677721Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:45.448687061Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:368"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-2-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:45.490208229Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:45.49844717Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:411"} {"msg":"Looking for unencrypted datasets in oxp_cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:45.639885672Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:496"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:45.64800172Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:512","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:45.648045251Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:429","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:45.648064571Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:414","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:45.648072091Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:491"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:45.648084242Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_3.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_3.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:45.658155983Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:45.658188774Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:366","id":"939962e4-70b1-47be-b2d6-938c818b62c4 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:45.769619074Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:304","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","zpool":"ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:45.77755525Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:348","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:45.77758691Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:350","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:45.77759494Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:354","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:45.777645511Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:356","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:45.777659931Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:45.777843284Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:45.777853185Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:368"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/3abf3a4f-8820-40af-a01f-4369b42a5529): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-08-05T16:42:45.781739981Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"3abf3a4f-8820-40af-a01f-4369b42a5529","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-3-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:45.822381804Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_939962e4-70b1-47be-b2d6-938c818b62c4/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:45.830826758Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:411"} {"msg":"Looking for unencrypted datasets in oxp_939962e4-70b1-47be-b2d6-938c818b62c4","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:45.975787149Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:496"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:45.983653963Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:512","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:45.983680534Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:429","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:45.983686754Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:414","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:45.983691234Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:491"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:45.983696214Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_4.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_4.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:45.993738425Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:45.993759296Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:366","id":"52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:46.106754413Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:304","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","zpool":"ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:46.115217027Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:348","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:46.115242528Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:350","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:46.115247868Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:354","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:46.115314339Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:356","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:46.115321939Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:46.115478741Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:46.115484782Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:368"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-4-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:46.158677078Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_52c33af8-e5fc-4cb6-93db-7a529f23be77/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:46.166453591Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:411"} {"msg":"Looking for unencrypted datasets in oxp_52c33af8-e5fc-4cb6-93db-7a529f23be77","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:46.308575414Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:496"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:46.316807065Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:512","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:46.316841486Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:429","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:46.316855826Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:414","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:46.316863276Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:491"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:46.316879756Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_5.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_5.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:46.326652843Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:46.326683723Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:366","id":"cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:46.440623186Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:304","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","zpool":"ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:46.448422579Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:348","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:46.44845808Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:350","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:46.44846436Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:354","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:46.44847046Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:356","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:46.448491451Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:46.448660183Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:46.448667894Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:368"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-5-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:46.493208813Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_cb14544a-cc03-4a04-84a7-da1d51f0a71d/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:46.501263431Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:411"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/3abf3a4f-8820-40af-a01f-4369b42a5529): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:42:46.52119804Z","hostname":"buskin","pid":1555,"component":"NexusNotifierTask","sled_id":"3abf3a4f-8820-40af-a01f-4369b42a5529","component":"SledAgent","file":"sled-agent/src/nexus.rs:601"} {"msg":"Looking for unencrypted datasets in oxp_cb14544a-cc03-4a04-84a7-da1d51f0a71d","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:46.646103171Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:496"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:46.653928804Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:512","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:46.653980625Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:429","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:46.653994985Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:414","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:46.654005505Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:491"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:46.654011105Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_6.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_6.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:46.663755512Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:46.663807823Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:366","id":"2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:46.779083198Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:304","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","zpool":"ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:46.787197327Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:348","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:46.787248038Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:350","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:46.787254618Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:354","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:46.787292109Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:356","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:46.787303809Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:46.787448931Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:46.787459661Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:368"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/3abf3a4f-8820-40af-a01f-4369b42a5529): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-08-05T16:42:46.791590762Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"3abf3a4f-8820-40af-a01f-4369b42a5529","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-6-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:46.828546672Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_2cb76428-77cf-4416-ab3e-f66fc49a03b4/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:46.836345035Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:411"} {"msg":"Looking for unencrypted datasets in oxp_2cb76428-77cf-4416-ab3e-f66fc49a03b4","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:46.982064889Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:496"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:46.989977914Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:512","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:46.990023325Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:429","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:46.990030875Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:414","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:46.990036155Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:491"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:46.990041815Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_7.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_7.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:47.000045996Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:47.000076086Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:366","id":"1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:47.110956907Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:304","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","zpool":"ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:47.119537014Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:348","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:47.119576324Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:350","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:47.119583995Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:354","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:47.119605135Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:356","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:47.119631495Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:47.119797958Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:47.119805618Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:368"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-7-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:47.160739056Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_1a42fe8d-3436-46f3-8051-0d2ff2a0be4f/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:47.168427297Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:411"} {"msg":"Looking for unencrypted datasets in oxp_1a42fe8d-3436-46f3-8051-0d2ff2a0be4f","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:47.317382368Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:496"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:47.326744097Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:512","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:47.326776538Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:429","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Managing disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:47.326782548Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:414","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Invoking Disk::new on an unmanaged disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:47.326789408Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:491"} {"msg":"Invoking SyntheticDisk::new","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:47.326793868Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/disk.rs:65","path":"/scratch/u2_8.vdev","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"GPT exists without Zpool: formatting zpool at /scratch/u2_8.vdev","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:47.336633456Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:360"} {"msg":"Formatting zpool with requested ID","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:47.336658826Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-hardware/src/disk.rs:366","id":"85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool)"} {"msg":"Ensuring zpool has datasets","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:47.453022541Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:304","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","zpool":"ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }"} {"msg":"Loading latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:47.461460425Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:348","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Loaded latest secret","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:47.461497526Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:350","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"Retrieving key","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:47.461504426Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:354","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"Got key","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:47.461587687Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:356","disk_id":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }","epoch":"0"} {"msg":"About to create keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:47.461602897Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:30","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Created keyfile","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:47.46174419Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:34","path":"Keypath(\"/var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key\")"} {"msg":"Ensuring encrypted filesystem: crypt for epoch 0","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:47.46175133Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:368"} {"msg":"Zeroed and unlinked keyfile /var/run/oxide/synthetic-vendor-synthetic-serial-8-synthetic-model-U2-zfs-aes-256-gcm.key","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:47.50401572Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/keyfile.rs:53"} {"msg":"Automatically destroying dataset: oxp_85b774a7-53a2-4f22-a027-1f642ba2d7f9/crypt/zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:47.512419604Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:411"} {"msg":"Looking for unencrypted datasets in oxp_85b774a7-53a2-4f22-a027-1f642ba2d7f9","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:47.655914531Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/dataset.rs:496"} {"msg":"Disk::new completed successfully","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:47.663586172Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:512","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Disk management started successfully","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:47.663627642Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:429","disk_identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Writing ledger to /pool/int/4576fedf-479b-43b3-8fad-bb8b7ec274f8/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:47.663715284Z","hostname":"buskin","pid":1555,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"Received storage manager update","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:47.663909337Z","hostname":"buskin","pid":1555,"component":"StorageMonitor","file":"sled-agent/src/storage_monitor.rs:93","disks":"AllDisks { generation: Generation(1), inner: AllDisksInner { values: {DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }, variant: M2, slot: 1024, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 4576fedf-479b-43b3-8fad-bb8b7ec274f8 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }: ImplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/m2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }, variant: M2, slot: 1025, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: d09b3f64-0a36-4052-8233-a94e62e662e2 (zpool), kind: Internal } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_0.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }, variant: U2, slot: 1026, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_1.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }, variant: U2, slot: 1027, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_2.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }, variant: U2, slot: 1028, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_3.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }, variant: U2, slot: 1029, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_4.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }, variant: U2, slot: 1030, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_5.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }, variant: U2, slot: 1031, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_6.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }, variant: U2, slot: 1032, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_7.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }, variant: U2, slot: 1033, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External } })), DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }: ExplicitlyManaged(Synthetic(SyntheticDisk { raw: RawSyntheticDisk { path: \"/scratch/u2_8.vdev\", identity: DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }, variant: U2, slot: 1034, firmware: DiskFirmware { active_slot: 1, next_active_slot: None, slot1_read_only: true, slot_firmware_versions: [Some(\"synthetic 1.0\")] } }, zpool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External } }))} }, mount_config: MountConfig { root: \"/\", synthetic_disk_root: \"/var/tmp\" } }"} {"msg":"Updated view of disks","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:47.663955008Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:580","dump_slices":"0","debug_datasets":"0","core_datasets":"0"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:47.663964398Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"Writing ledger to /pool/int/d09b3f64-0a36-4052-8233-a94e62e662e2/config/.omicron-physical-disks.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:47.66408507Z","hostname":"buskin","pid":1555,"request":"omicron_physical_disks_ensure","component":"StorageManager","file":"common/src/ledger.rs:199"} {"msg":"physical disks ensure: Updated storage","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:47.664323794Z","hostname":"buskin","pid":1555,"sled_id":"3abf3a4f-8820-40af-a01f-4369b42a5529","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:846"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:47.664339715Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"physical disks ensure: Propagating new generation of disks","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:47.664343745Z","hostname":"buskin","pid":1555,"sled_id":"3abf3a4f-8820-40af-a01f-4369b42a5529","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:872","generation":"Generation(1)"} {"msg":"physical disks ensure: Updated storage monitor","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:47.664348265Z","hostname":"buskin","pid":1555,"sled_id":"3abf3a4f-8820-40af-a01f-4369b42a5529","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:877"} {"msg":"physical disks ensure: Updated zone bundler","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:47.664353285Z","hostname":"buskin","pid":1555,"sled_id":"3abf3a4f-8820-40af-a01f-4369b42a5529","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:882"} {"msg":"use_only_these_disks: Processing new request","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:47.664363415Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"3abf3a4f-8820-40af-a01f-4369b42a5529","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:116","gen":"Generation(1)"} {"msg":"physical disks ensure: Updated probes","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:47.664376915Z","hostname":"buskin","pid":1555,"sled_id":"3abf3a4f-8820-40af-a01f-4369b42a5529","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:888"} {"msg":"use_only_these_disks: Processing new request","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:47.664424286Z","hostname":"buskin","pid":1555,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:838","gen":"Generation(1)"} {"msg":"physical disks ensure: Updated instances","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:47.664431646Z","hostname":"buskin","pid":1555,"sled_id":"3abf3a4f-8820-40af-a01f-4369b42a5529","component":"SledAgent","file":"sled-agent/src/sled_agent.rs:893"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:47.664512148Z","hostname":"buskin","pid":1555,"uri":"/omicron-physical-disks","method":"PUT","req_id":"44d6dcac-33c1-4c60-a33d-f052d3edf127","remote_addr":"[fd00:1122:3344:101::1]:43904","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":12245116,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(2), zones: [OmicronZoneConfig { id: 83a194e2-3464-4a84-ae47-e33a0983bc56, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: fe17df80-010a-4b68-9ebc-6a94437be875, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:47.695212621Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:47.695385204Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:55797"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83a194e2-3464-4a84-ae47-e33a0983bc56, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:47.695599058Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:47.695604978Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 83a194e2-3464-4a84-ae47-e33a0983bc56, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:47.757200658Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:47.757239359Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/3abf3a4f-8820-40af-a01f-4369b42a5529): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-08-05T16:42:47.79423806Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"3abf3a4f-8820-40af-a01f-4369b42a5529","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe17df80-010a-4b68-9ebc-6a94437be875, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:47.818558885Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:47.818588445Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: fe17df80-010a-4b68-9ebc-6a94437be875, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:47.879095637Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"No ledger in /pool/int/4576fedf-479b-43b3-8fad-bb8b7ec274f8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:47.879128357Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"No ledger in /pool/int/d09b3f64-0a36-4052-8233-a94e62e662e2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:47.879138478Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:188"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:47.879182718Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:47.923441913Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:47.923470854Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:47.923474984Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_fe17df80-010a-4b68-9ebc-6a94437be875","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:47.975241996Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_83a194e2-3464-4a84-ae47-e33a0983bc56","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:47.978762106Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_internal_dns_5145c3cc-86a8-4ca7-a5dd-d58654518f19","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:47.982435769Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_internal_dns_fe17df80-010a-4b68-9ebc-6a94437be875","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:47.994127128Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_internal_dns_83a194e2-3464-4a84-ae47-e33a0983bc56","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:47.998109196Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_internal_dns_5145c3cc-86a8-4ca7-a5dd-d58654518f19","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:48.002051843Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/3abf3a4f-8820-40af-a01f-4369b42a5529): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:42:48.521635664Z","hostname":"buskin","pid":1555,"component":"NexusNotifierTask","sled_id":"3abf3a4f-8820-40af-a01f-4369b42a5529","component":"SledAgent","file":"sled-agent/src/nexus.rs:601"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/3abf3a4f-8820-40af-a01f-4369b42a5529): error trying to connect: dns error: proto error: io error: No route to host (os error 148)","v":0,"name":"SledAgent","level":50,"time":"2024-08-05T16:42:48.797154723Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"3abf3a4f-8820-40af-a01f-4369b42a5529","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"Profile for oxz_internal_dns_83a194e2-3464-4a84-ae47-e33a0983bc56:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:49.473997655Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:49.474046076Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:1::, width: 64 }"} {"msg":"Profile for oxz_internal_dns_fe17df80-010a-4b68-9ebc-6a94437be875:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:49.950064284Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_internal_dns_83a194e2-3464-4a84-ae47-e33a0983bc56 zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:49.950157285Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_83a194e2-3464-4a84-ae47-e33a0983bc56","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:472"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:50.433359775Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:3::, width: 64 }"} {"msg":"Profile for oxz_internal_dns_5145c3cc-86a8-4ca7-a5dd-d58654518f19:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:50.433476167Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Sending prefix to ddmd for advertisement","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:50.433486197Z","hostname":"buskin","pid":1555,"DdmAdminClient":"[::1]:8000","component":"BootstrapAgentStartup","file":"clients/ddm-admin-client/src/lib.rs:86","prefix":"Ipv6Net { addr: fd00:1122:3344:2::, width: 64 }"} {"msg":"Booting oxz_internal_dns_fe17df80-010a-4b68-9ebc-6a94437be875 zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:50.433534868Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_fe17df80-010a-4b68-9ebc-6a94437be875","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:472"} {"msg":"wait for service Some(\"oxz_internal_dns_83a194e2-3464-4a84-ae47-e33a0983bc56\") failed: Property not found. retry in 50.548963ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:42:50.446812224Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_83a194e2-3464-4a84-ae47-e33a0983bc56","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Booting oxz_internal_dns_5145c3cc-86a8-4ca7-a5dd-d58654518f19 zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:50.446858685Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_5145c3cc-86a8-4ca7-a5dd-d58654518f19","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:472"} {"msg":"wait for service Some(\"oxz_internal_dns_83a194e2-3464-4a84-ae47-e33a0983bc56\") failed: Property not found. retry in 79.821833ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:42:50.507116232Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_83a194e2-3464-4a84-ae47-e33a0983bc56","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_fe17df80-010a-4b68-9ebc-6a94437be875\") failed: Property not found. retry in 55.827991ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:42:50.56618542Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_fe17df80-010a-4b68-9ebc-6a94437be875","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_5145c3cc-86a8-4ca7-a5dd-d58654518f19\") failed: Property not found. retry in 32.606347ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:42:50.573250491Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_5145c3cc-86a8-4ca7-a5dd-d58654518f19","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_83a194e2-3464-4a84-ae47-e33a0983bc56\") failed: Property not found. retry in 120.092936ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:42:50.594612495Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_83a194e2-3464-4a84-ae47-e33a0983bc56","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_5145c3cc-86a8-4ca7-a5dd-d58654518f19\") failed: Property not found. retry in 113.558994ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:42:50.612787995Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_5145c3cc-86a8-4ca7-a5dd-d58654518f19","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_fe17df80-010a-4b68-9ebc-6a94437be875\") failed: Property not found. retry in 133.6297ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:42:50.639302137Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_fe17df80-010a-4b68-9ebc-6a94437be875","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_83a194e2-3464-4a84-ae47-e33a0983bc56\") failed: Property not found. retry in 279.161549ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:42:50.775637882Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_83a194e2-3464-4a84-ae47-e33a0983bc56","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_fe17df80-010a-4b68-9ebc-6a94437be875\") failed: Property not found. retry in 270.398765ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:42:50.899527765Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_fe17df80-010a-4b68-9ebc-6a94437be875","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_5145c3cc-86a8-4ca7-a5dd-d58654518f19\") failed: Property not found. retry in 144.53606ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:42:50.914428669Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_5145c3cc-86a8-4ca7-a5dd-d58654518f19","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_83a194e2-3464-4a84-ae47-e33a0983bc56\") failed: Property not found. retry in 427.126483ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:42:51.085029197Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_83a194e2-3464-4a84-ae47-e33a0983bc56","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_5145c3cc-86a8-4ca7-a5dd-d58654518f19\") failed: Property not found. retry in 229.582987ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:42:51.112135649Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_5145c3cc-86a8-4ca7-a5dd-d58654518f19","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_fe17df80-010a-4b68-9ebc-6a94437be875\") failed: Property not found. retry in 241.053817ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:42:51.181138736Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_fe17df80-010a-4b68-9ebc-6a94437be875","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_5145c3cc-86a8-4ca7-a5dd-d58654518f19\") failed: Property not found. retry in 709.904483ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:42:51.350769299Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_5145c3cc-86a8-4ca7-a5dd-d58654518f19","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_fe17df80-010a-4b68-9ebc-6a94437be875\") failed: Property not found. retry in 736.942138ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:42:51.430453818Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_fe17df80-010a-4b68-9ebc-6a94437be875","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_83a194e2-3464-4a84-ae47-e33a0983bc56\") failed: Property not found. retry in 971.009818ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:42:51.520449033Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_83a194e2-3464-4a84-ae47-e33a0983bc56","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_5145c3cc-86a8-4ca7-a5dd-d58654518f19\") failed: Property not found. retry in 966.063975ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:42:52.073407802Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_5145c3cc-86a8-4ca7-a5dd-d58654518f19","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_fe17df80-010a-4b68-9ebc-6a94437be875\") failed: Property not found. retry in 781.110945ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:42:52.177852563Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_fe17df80-010a-4b68-9ebc-6a94437be875","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_83a194e2-3464-4a84-ae47-e33a0983bc56\") failed: Property not found. retry in 734.470834ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:42:52.501628745Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_83a194e2-3464-4a84-ae47-e33a0983bc56","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_fe17df80-010a-4b68-9ebc-6a94437be875\") failed: Property not found. retry in 743.757803ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:42:52.968580947Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_fe17df80-010a-4b68-9ebc-6a94437be875","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_5145c3cc-86a8-4ca7-a5dd-d58654518f19\") failed: Property not found. retry in 893.599751ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:42:53.048953978Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_5145c3cc-86a8-4ca7-a5dd-d58654518f19","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_83a194e2-3464-4a84-ae47-e33a0983bc56\") failed: Property not found. retry in 1.46543435s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:42:53.244455672Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_83a194e2-3464-4a84-ae47-e33a0983bc56","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_fe17df80-010a-4b68-9ebc-6a94437be875\") failed: Property not found. retry in 794.176725ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:42:53.721119261Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_fe17df80-010a-4b68-9ebc-6a94437be875","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Ensuring address fd00:1122:3344:101::2 exists - OK","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:53.819651231Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3946"} {"msg":"Ensuring there is a default route","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:53.819700312Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3956","gateway":"fd00:1122:3344:101::1"} {"msg":"configuring dendrite service","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:53.825477691Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:4035"} {"msg":"refreshed dendrite service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:53.953020616Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:4083"} {"msg":"configuring lldp service","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:53.953058926Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:4110"} {"msg":"wait for service Some(\"oxz_internal_dns_5145c3cc-86a8-4ca7-a5dd-d58654518f19\") failed: Property not found. retry in 1.206315818s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:42:53.955718162Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_5145c3cc-86a8-4ca7-a5dd-d58654518f19","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"refreshed lldpd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:53.997052916Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:4123"} {"msg":"configuring MGS service","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:53.997093337Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3987"} {"msg":"refreshed MGS service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:54.054974464Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:4029"} {"msg":"configuring wicketd","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:54.055016794Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:4090","rack_subnet":"fd00:1122:3344::"} {"msg":"refreshed wicketd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:54.074930304Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:4101"} {"msg":"configuring mgd service","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:54.074966005Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:4143"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:54.101232322Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::2]:52596"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:54.101537498Z","hostname":"buskin","pid":1555,"uri":"/sled-identifiers","method":"GET","req_id":"5c1164d3-2727-41e2-9a82-cc505746da64","remote_addr":"[fd00:1122:3344:101::2]:52596","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":85,"response_code":"200"} {"msg":"refreshed mgd service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:54.162701411Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:4175"} {"msg":"configuring mg-ddm service","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:54.162735981Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:4181"} {"msg":"refreshed mg-ddm service with new configuration","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:54.27110951Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:4222"} {"msg":"Determining physical location of our switch zone at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:54.27115803Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:366"} {"msg":"Initializing 1 Uplinks on Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:54.303420341Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:414"} {"msg":"Building Port Configuration","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:54.335371175Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:675"} {"msg":"Checking dendrite uptime","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:54.335472677Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:713"} {"msg":"Dendrite online","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:54.336021706Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:716","uptime":"0"} {"msg":"Configuring default uplink on switch","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:54.336032317Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:436","config":"PortSettings {\n links: {\n \"0\": LinkSettings {\n addrs: [\n 10.151.1.174,\n ],\n params: LinkCreate {\n autoneg: false,\n fec: None,\n kr: false,\n lane: Some(\n LinkId(\n 0,\n ),\n ),\n speed: Speed40G,\n },\n },\n },\n}"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:54.371869538Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3823"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:54.391203778Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3834"} {"msg":"Received DetectedRawDisksChanged { raw_disks: {}, tx: ... }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:54.408908129Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:54.408973031Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:601","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:54.408984381Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:622","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-0\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:54.408996501Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:601","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:54.409004021Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:622","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-M2\", serial: \"synthetic-serial-1\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:54.409011171Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:601","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:54.409026931Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:622","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-0\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:54.409034292Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:601","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:54.409040962Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:622","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-1\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:54.409049592Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:601","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:54.409056772Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:622","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-2\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:54.409063882Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:601","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:54.409070662Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:622","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-3\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:54.409079692Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:601","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:54.409086923Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:622","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-4\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:54.409091873Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:601","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:54.409096773Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:622","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-5\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:54.409101653Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:601","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:54.409106703Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:622","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-6\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:54.409121923Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:601","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:54.409127313Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:622","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-7\" }"} {"msg":"Removing disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:54.409132143Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:601","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"Not removing synthetic disk","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:54.409137144Z","hostname":"buskin","pid":1555,"component":"StorageResources","file":"sled-storage/src/resources.rs:622","identity":"DiskIdentity { vendor: \"synthetic-vendor\", model: \"synthetic-model-U2\", serial: \"synthetic-serial-8\" }"} {"msg":"wait for service Some(\"oxz_internal_dns_fe17df80-010a-4b68-9ebc-6a94437be875\") failed: Property not found. retry in 1.347197649s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:42:54.525581169Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_fe17df80-010a-4b68-9ebc-6a94437be875","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_83a194e2-3464-4a84-ae47-e33a0983bc56\") failed: Property not found. retry in 903.3388ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:42:54.7185472Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_83a194e2-3464-4a84-ae47-e33a0983bc56","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_internal_dns_5145c3cc-86a8-4ca7-a5dd-d58654518f19\") failed: Property not found. retry in 1.241292377s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:42:55.171109567Z","hostname":"buskin","pid":1555,"zone":"oxz_internal_dns_5145c3cc-86a8-4ca7-a5dd-d58654518f19","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:56.429538027Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3061","zone":"oxz_internal_dns_fe17df80-010a-4b68-9ebc-6a94437be875"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:56.429603599Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3061","zone":"oxz_internal_dns_83a194e2-3464-4a84-ae47-e33a0983bc56"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:56.429609659Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3061","zone":"oxz_internal_dns_5145c3cc-86a8-4ca7-a5dd-d58654518f19"} {"msg":"Writing ledger to /pool/int/4576fedf-479b-43b3-8fad-bb8b7ec274f8/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:56.429620019Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/d09b3f64-0a36-4052-8233-a94e62e662e2/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:56.429962215Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:56.43029045Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"a6b6629a-566b-4f96-ac03-be2d8844e230","remote_addr":"[fd00:1122:3344:101::1]:55797","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":8734834,"response_code":"204"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:56.431823146Z","hostname":"buskin","pid":1555,"dns_config_addr":"[fd00:1122:3344:1::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:555"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:56.484631977Z","hostname":"buskin","pid":1555,"dns_config_addr":"[fd00:1122:3344:2::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:555"} {"msg":"Configuring DNS server","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:56.536657174Z","hostname":"buskin","pid":1555,"dns_config_addr":"[fd00:1122:3344:3::1]:5353","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:555"} {"msg":"Configured all DNS servers","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:56.587939179Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:591"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:56.587954889Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:260"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:56.619969045Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:277","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:56.620671187Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:302"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:56.620678007Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/bootstrap/early_networking.rs:194","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(3), zones: [OmicronZoneConfig { id: 83a194e2-3464-4a84-ae47-e33a0983bc56, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: fe17df80-010a-4b68-9ebc-6a94437be875, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5b6cee62-004c-4843-9e16-c25d2b0c743b, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6300d678-c957-4259-9a3e-c167ee4d0fb0, kind: Service { id: 5b6cee62-004c-4843-9e16-c25d2b0c743b }, name: Name(\"ntp-5b6cee62-004c-4843-9e16-c25d2b0c743b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 71])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:56.649329476Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:56.649443117Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:63984"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83a194e2-3464-4a84-ae47-e33a0983bc56, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:56.649655721Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:56.649662491Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 83a194e2-3464-4a84-ae47-e33a0983bc56, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:56.670471816Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:56.670508897Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe17df80-010a-4b68-9ebc-6a94437be875, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:56.687987295Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:56.688029226Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: fe17df80-010a-4b68-9ebc-6a94437be875, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:56.70530992Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Reading ledger from /pool/int/4576fedf-479b-43b3-8fad-bb8b7ec274f8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:56.705367951Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/d09b3f64-0a36-4052-8233-a94e62e662e2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:56.705555065Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:56.705672947Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:56.716802146Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:260"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:56.749078977Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:277","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:56.749713438Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:302"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:56.749721308Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:194","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:56.77858916Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:426","port":"Port { inner: PortInner(PortData { name: \"opte0\", ip: 172.30.3.5, mac: MacAddr6([168, 64, 37, 255, 247, 71]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.3.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:56.77859747Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:1210","zone_type":"boundary_ntp"} {"msg":"no nat entry found for: 10.151.1.169","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:56.779066948Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.169","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:56.779082258Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"creation of nat entry successful for: 10.151.1.169","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:56.779426544Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:56.780676886Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Configuring new Omicron zone: oxz_ntp_5b6cee62-004c-4843-9e16-c25d2b0c743b","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:56.804457461Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_ntp_5b6cee62-004c-4843-9e16-c25d2b0c743b","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:56.822307905Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_ntp_5b6cee62-004c-4843-9e16-c25d2b0c743b:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:57.441951272Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_ntp_5b6cee62-004c-4843-9e16-c25d2b0c743b zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:42:57.442126315Z","hostname":"buskin","pid":1555,"zone":"oxz_ntp_5b6cee62-004c-4843-9e16-c25d2b0c743b","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:472"} {"msg":"wait for service Some(\"oxz_ntp_5b6cee62-004c-4843-9e16-c25d2b0c743b\") failed: Property not found. retry in 67.623322ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:42:57.575482459Z","hostname":"buskin","pid":1555,"zone":"oxz_ntp_5b6cee62-004c-4843-9e16-c25d2b0c743b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_5b6cee62-004c-4843-9e16-c25d2b0c743b\") failed: Property not found. retry in 118.200897ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:42:57.654878963Z","hostname":"buskin","pid":1555,"zone":"oxz_ntp_5b6cee62-004c-4843-9e16-c25d2b0c743b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_5b6cee62-004c-4843-9e16-c25d2b0c743b\") failed: Property not found. retry in 240.495868ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:42:57.804710509Z","hostname":"buskin","pid":1555,"zone":"oxz_ntp_5b6cee62-004c-4843-9e16-c25d2b0c743b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_5b6cee62-004c-4843-9e16-c25d2b0c743b\") failed: Property not found. retry in 274.425397ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:42:58.054901015Z","hostname":"buskin","pid":1555,"zone":"oxz_ntp_5b6cee62-004c-4843-9e16-c25d2b0c743b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_5b6cee62-004c-4843-9e16-c25d2b0c743b\") failed: Property not found. retry in 841.624352ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:42:58.339208473Z","hostname":"buskin","pid":1555,"zone":"oxz_ntp_5b6cee62-004c-4843-9e16-c25d2b0c743b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_5b6cee62-004c-4843-9e16-c25d2b0c743b\") failed: Property not found. retry in 1.014178632s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:42:59.190827355Z","hostname":"buskin","pid":1555,"zone":"oxz_ntp_5b6cee62-004c-4843-9e16-c25d2b0c743b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_5b6cee62-004c-4843-9e16-c25d2b0c743b\") failed: Property not found. retry in 1.105422431s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:00.214102576Z","hostname":"buskin","pid":1555,"zone":"oxz_ntp_5b6cee62-004c-4843-9e16-c25d2b0c743b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_5b6cee62-004c-4843-9e16-c25d2b0c743b\") failed: Property not found. retry in 965.86147ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:01.330426002Z","hostname":"buskin","pid":1555,"zone":"oxz_ntp_5b6cee62-004c-4843-9e16-c25d2b0c743b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_ntp_5b6cee62-004c-4843-9e16-c25d2b0c743b\") failed: Property not found. retry in 741.660638ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:02.306642189Z","hostname":"buskin","pid":1555,"zone":"oxz_ntp_5b6cee62-004c-4843-9e16-c25d2b0c743b","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:03.068840607Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3061","zone":"oxz_ntp_5b6cee62-004c-4843-9e16-c25d2b0c743b"} {"msg":"Writing ledger to /pool/int/4576fedf-479b-43b3-8fad-bb8b7ec274f8/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:03.068901418Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/d09b3f64-0a36-4052-8233-a94e62e662e2/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:03.069259234Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:03.069536589Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"96112238-7f5c-46cf-b341-847bf29225df","remote_addr":"[fd00:1122:3344:101::1]:63984","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":6420061,"response_code":"204"} {"msg":"Waiting for rack time synchronization","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:03.072000301Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:632"} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:03.101581906Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:612"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:03.101762889Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:60121"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_5b6cee62-004c-4843-9e16-c25d2b0c743b': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-08-05T16:43:03.111144369Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3650"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:03.111301191Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"cdce478e-b658-4641-a44c-17103bcf5280","remote_addr":"[fd00:1122:3344:101::1]:60121","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":9439,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:03.114075979Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:662","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:03.223704728Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:612"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:03.22383217Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:43024"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_5b6cee62-004c-4843-9e16-c25d2b0c743b': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-08-05T16:43:07.232649433Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3650"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:07.232750545Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"22887bbc-a860-4b20-ad34-a3321f55e0df","remote_addr":"[fd00:1122:3344:101::1]:43024","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":4008815,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:07.235936219Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:662","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:07.347787056Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:612"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:07.347905189Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:65434"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_5b6cee62-004c-4843-9e16-c25d2b0c743b': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-08-05T16:43:07.357264768Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3650"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:07.357434461Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"657298ce-b31a-4e61-9605-b9ddb8f12b3d","remote_addr":"[fd00:1122:3344:101::1]:65434","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":9409,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:07.359505986Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:662","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:07.51215383Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:612"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:07.512273361Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:47146"} {"msg":"chronyc command failed: Error running command in zone 'oxz_ntp_5b6cee62-004c-4843-9e16-c25d2b0c743b': Command [/usr/bin/chronyc -c tracking] executed and failed with status: exit status: 1 stdout: 506 Cannot talk to daemon\n stderr: ","v":0,"name":"SledAgent","level":50,"time":"2024-08-05T16:43:10.656935017Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3650"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:10.657164414Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"9a4d791f-780a-4f29-9b1a-8a7e8504fd08","remote_addr":"[fd00:1122:3344:101::1]:47146","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"NTP zone not ready","latency_us":4008670,"response_code":"500"} {"msg":"Time is not yet synchronized","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:10.659540954Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:662","error":"\"Time is synchronized on 0/1 sleds\""} {"msg":"Checking time synchronization for [fd00:1122:3344:101::1]:12345...","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:10.832851409Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:612"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:10.832997302Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:39239"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:10.853288198Z","hostname":"buskin","pid":1555,"uri":"/timesync","method":"GET","req_id":"81d65d02-aa49-4309-b37e-720cf1b37677","remote_addr":"[fd00:1122:3344:101::1]:39239","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":20159,"response_code":"200"} {"msg":"Timesync for [fd00:1122:3344:101::1]:12345 TimeSync { sync: true, ref_id: 2728380539, ip_addr: 162.159.200.123, stratum: 4, ref_time: 1722876189.9402506, correction: 0.0 }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:10.857030877Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:640"} {"msg":"Finished setting up Internal DNS and NTP","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:10.857043093Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1230"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(4), zones: [OmicronZoneConfig { id: 83a194e2-3464-4a84-ae47-e33a0983bc56, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: fe17df80-010a-4b68-9ebc-6a94437be875, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5b6cee62-004c-4843-9e16-c25d2b0c743b, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6300d678-c957-4259-9a3e-c167ee4d0fb0, kind: Service { id: 5b6cee62-004c-4843-9e16-c25d2b0c743b }, name: Name(\"ntp-5b6cee62-004c-4843-9e16-c25d2b0c743b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 71])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 3c470f26-8422-4139-9f7f-81a0d84becba, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: acae5a41-5da9-420f-9bc8-154e4abdaf87, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } } } }, OmicronZoneConfig { id: eed4ad2f-7765-43e1-8943-ea458ed22e57, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } } } }, OmicronZoneConfig { id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 902ea001-1913-4130-9979-87a8d3db1f53, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:10.88746017Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:10.887610883Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:55017"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83a194e2-3464-4a84-ae47-e33a0983bc56, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:10.887964369Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:10.887970229Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 83a194e2-3464-4a84-ae47-e33a0983bc56, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:10.909782661Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:10.909822502Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe17df80-010a-4b68-9ebc-6a94437be875, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:10.928643433Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:10.928689793Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: fe17df80-010a-4b68-9ebc-6a94437be875, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c470f26-8422-4139-9f7f-81a0d84becba, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:10.946315864Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:10.946359195Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 3c470f26-8422-4139-9f7f-81a0d84becba, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: acae5a41-5da9-420f-9bc8-154e4abdaf87, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:11.008834191Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:11.008872938Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: acae5a41-5da9-420f-9bc8-154e4abdaf87, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eed4ad2f-7765-43e1-8943-ea458ed22e57, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:11.071219844Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:11.071258285Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: eed4ad2f-7765-43e1-8943-ea458ed22e57, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:11.132160132Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:11.132202253Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 902ea001-1913-4130-9979-87a8d3db1f53, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:11.194365353Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:11.194407094Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 902ea001-1913-4130-9979-87a8d3db1f53, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:11.262431572Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Reading ledger from /pool/int/4576fedf-479b-43b3-8fad-bb8b7ec274f8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:11.262479048Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/d09b3f64-0a36-4052-8233-a94e62e662e2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:11.262669242Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:11.262829266Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:11.346554788Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:11.346614719Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:11.346619629Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:11.346625789Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:11.346633249Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_3c470f26-8422-4139-9f7f-81a0d84becba","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:11.436266687Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_902ea001-1913-4130-9979-87a8d3db1f53","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:11.440236975Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_acae5a41-5da9-420f-9bc8-154e4abdaf87","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:11.444437287Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_eed4ad2f-7765-43e1-8943-ea458ed22e57","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:11.448336833Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_cockroachdb_b2f11b30-44fe-4b76-98c0-1891b1b0971a","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:11.452576965Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_cockroachdb_3c470f26-8422-4139-9f7f-81a0d84becba","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:11.45750164Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_902ea001-1913-4130-9979-87a8d3db1f53","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:11.462186789Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_acae5a41-5da9-420f-9bc8-154e4abdaf87","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:11.466479068Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_eed4ad2f-7765-43e1-8943-ea458ed22e57","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:11.470980449Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_cockroachdb_b2f11b30-44fe-4b76-98c0-1891b1b0971a","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:11.475905873Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:13.61479006Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","component":"MetricsManager","sled_id":"3abf3a4f-8820-40af-a01f-4369b42a5529","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"374.536001ms"} {"msg":"Profile for oxz_cockroachdb_3c470f26-8422-4139-9f7f-81a0d84becba:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:14.351915137Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_3c470f26-8422-4139-9f7f-81a0d84becba zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:14.352050814Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_3c470f26-8422-4139-9f7f-81a0d84becba","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:472"} {"msg":"Profile for oxz_cockroachdb_eed4ad2f-7765-43e1-8943-ea458ed22e57:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:14.366080423Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_eed4ad2f-7765-43e1-8943-ea458ed22e57 zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:14.366245707Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_eed4ad2f-7765-43e1-8943-ea458ed22e57","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:472"} {"msg":"Profile for oxz_cockroachdb_acae5a41-5da9-420f-9bc8-154e4abdaf87:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:14.395024602Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_acae5a41-5da9-420f-9bc8-154e4abdaf87 zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:14.395128882Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_acae5a41-5da9-420f-9bc8-154e4abdaf87","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:472"} {"msg":"Profile for oxz_cockroachdb_902ea001-1913-4130-9979-87a8d3db1f53:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:14.40064889Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_902ea001-1913-4130-9979-87a8d3db1f53 zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:14.400787484Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_902ea001-1913-4130-9979-87a8d3db1f53","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:472"} {"msg":"Profile for oxz_cockroachdb_b2f11b30-44fe-4b76-98c0-1891b1b0971a:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:14.494066065Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_cockroachdb_b2f11b30-44fe-4b76-98c0-1891b1b0971a zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:14.494209758Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_b2f11b30-44fe-4b76-98c0-1891b1b0971a","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:472"} {"msg":"wait for service Some(\"oxz_cockroachdb_3c470f26-8422-4139-9f7f-81a0d84becba\") failed: Property not found. retry in 26.780164ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:14.507582884Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_3c470f26-8422-4139-9f7f-81a0d84becba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_eed4ad2f-7765-43e1-8943-ea458ed22e57\") failed: Property not found. retry in 30.398258ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:14.515189306Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_eed4ad2f-7765-43e1-8943-ea458ed22e57","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_acae5a41-5da9-420f-9bc8-154e4abdaf87\") failed: Property not found. retry in 65.52641ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:14.531563927Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_acae5a41-5da9-420f-9bc8-154e4abdaf87","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_902ea001-1913-4130-9979-87a8d3db1f53\") failed: Property not found. retry in 65.553682ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:14.539331771Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_902ea001-1913-4130-9979-87a8d3db1f53","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3c470f26-8422-4139-9f7f-81a0d84becba\") failed: Property not found. retry in 89.995037ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:14.545940855Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_3c470f26-8422-4139-9f7f-81a0d84becba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_eed4ad2f-7765-43e1-8943-ea458ed22e57\") failed: Property not found. retry in 57.71122ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:14.554390567Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_eed4ad2f-7765-43e1-8943-ea458ed22e57","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_acae5a41-5da9-420f-9bc8-154e4abdaf87\") failed: Property not found. retry in 60.791258ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:14.757561208Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_acae5a41-5da9-420f-9bc8-154e4abdaf87","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3c470f26-8422-4139-9f7f-81a0d84becba\") failed: Property not found. retry in 112.498824ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:15.021930933Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_3c470f26-8422-4139-9f7f-81a0d84becba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_902ea001-1913-4130-9979-87a8d3db1f53\") failed: Property not found. retry in 145.267808ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:15.23093494Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_902ea001-1913-4130-9979-87a8d3db1f53","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_acae5a41-5da9-420f-9bc8-154e4abdaf87\") failed: Property not found. retry in 203.023334ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:15.531806644Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_acae5a41-5da9-420f-9bc8-154e4abdaf87","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_eed4ad2f-7765-43e1-8943-ea458ed22e57\") failed: Property not found. retry in 276.169191ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:15.594052613Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_eed4ad2f-7765-43e1-8943-ea458ed22e57","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3c470f26-8422-4139-9f7f-81a0d84becba\") failed: Property not found. retry in 228.009776ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:15.761870124Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_3c470f26-8422-4139-9f7f-81a0d84becba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_902ea001-1913-4130-9979-87a8d3db1f53\") failed: Property not found. retry in 238.382755ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:15.974206153Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_902ea001-1913-4130-9979-87a8d3db1f53","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_acae5a41-5da9-420f-9bc8-154e4abdaf87\") failed: Property not found. retry in 401.283721ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:16.093807786Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_acae5a41-5da9-420f-9bc8-154e4abdaf87","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_eed4ad2f-7765-43e1-8943-ea458ed22e57\") failed: Property not found. retry in 398.782537ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:16.287855059Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_eed4ad2f-7765-43e1-8943-ea458ed22e57","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b2f11b30-44fe-4b76-98c0-1891b1b0971a\") failed: Property not found. retry in 61.970725ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:16.550945445Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_b2f11b30-44fe-4b76-98c0-1891b1b0971a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3c470f26-8422-4139-9f7f-81a0d84becba\") failed: Property not found. retry in 494.863254ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:16.679995124Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_3c470f26-8422-4139-9f7f-81a0d84becba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_902ea001-1913-4130-9979-87a8d3db1f53\") failed: Property not found. retry in 236.825194ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:16.724719063Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_902ea001-1913-4130-9979-87a8d3db1f53","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_acae5a41-5da9-420f-9bc8-154e4abdaf87\") failed: Property not found. retry in 837.212067ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:16.747054232Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_acae5a41-5da9-420f-9bc8-154e4abdaf87","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_eed4ad2f-7765-43e1-8943-ea458ed22e57\") failed: Property not found. retry in 756.133841ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:16.761941088Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_eed4ad2f-7765-43e1-8943-ea458ed22e57","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b2f11b30-44fe-4b76-98c0-1891b1b0971a\") failed: Property not found. retry in 125.567175ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:16.782282946Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_b2f11b30-44fe-4b76-98c0-1891b1b0971a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b2f11b30-44fe-4b76-98c0-1891b1b0971a\") failed: Property not found. retry in 278.82169ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:16.92767732Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_b2f11b30-44fe-4b76-98c0-1891b1b0971a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_902ea001-1913-4130-9979-87a8d3db1f53\") failed: Property not found. retry in 1.065798478s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:16.977838674Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_902ea001-1913-4130-9979-87a8d3db1f53","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3c470f26-8422-4139-9f7f-81a0d84becba\") failed: Property not found. retry in 1.267241012s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:17.16205476Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_3c470f26-8422-4139-9f7f-81a0d84becba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b2f11b30-44fe-4b76-98c0-1891b1b0971a\") failed: Property not found. retry in 210.585035ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:17.217468499Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_b2f11b30-44fe-4b76-98c0-1891b1b0971a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b2f11b30-44fe-4b76-98c0-1891b1b0971a\") failed: Property not found. retry in 762.980308ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:17.44795813Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_b2f11b30-44fe-4b76-98c0-1891b1b0971a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_eed4ad2f-7765-43e1-8943-ea458ed22e57\") failed: Property not found. retry in 519.254827ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:17.5345895Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_eed4ad2f-7765-43e1-8943-ea458ed22e57","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_acae5a41-5da9-420f-9bc8-154e4abdaf87\") failed: Property not found. retry in 605.333726ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:17.600092586Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_acae5a41-5da9-420f-9bc8-154e4abdaf87","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_902ea001-1913-4130-9979-87a8d3db1f53\") failed: Property not found. retry in 1.021059132s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:18.05759499Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_902ea001-1913-4130-9979-87a8d3db1f53","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_eed4ad2f-7765-43e1-8943-ea458ed22e57\") failed: Property not found. retry in 608.389297ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:18.065603115Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_eed4ad2f-7765-43e1-8943-ea458ed22e57","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_acae5a41-5da9-420f-9bc8-154e4abdaf87\") failed: Property not found. retry in 1.118393339s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:18.218146925Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_acae5a41-5da9-420f-9bc8-154e4abdaf87","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b2f11b30-44fe-4b76-98c0-1891b1b0971a\") failed: Property not found. retry in 763.027699ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:18.22798564Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_b2f11b30-44fe-4b76-98c0-1891b1b0971a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3c470f26-8422-4139-9f7f-81a0d84becba\") failed: Property not found. retry in 1.021733309s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:18.443556282Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_3c470f26-8422-4139-9f7f-81a0d84becba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_eed4ad2f-7765-43e1-8943-ea458ed22e57\") failed: Property not found. retry in 894.641188ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:18.685545875Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_eed4ad2f-7765-43e1-8943-ea458ed22e57","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b2f11b30-44fe-4b76-98c0-1891b1b0971a\") failed: Property not found. retry in 888.055016ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:19.004478387Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_b2f11b30-44fe-4b76-98c0-1891b1b0971a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_902ea001-1913-4130-9979-87a8d3db1f53\") failed: Property not found. retry in 578.393447ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:19.091856063Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_902ea001-1913-4130-9979-87a8d3db1f53","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_acae5a41-5da9-420f-9bc8-154e4abdaf87\") failed: Property not found. retry in 928.379446ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:19.349841393Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_acae5a41-5da9-420f-9bc8-154e4abdaf87","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3c470f26-8422-4139-9f7f-81a0d84becba\") failed: Property not found. retry in 1.220228438s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:19.477863159Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_3c470f26-8422-4139-9f7f-81a0d84becba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_eed4ad2f-7765-43e1-8943-ea458ed22e57\") failed: Property not found. retry in 614.766779ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:19.593302865Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_eed4ad2f-7765-43e1-8943-ea458ed22e57","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_902ea001-1913-4130-9979-87a8d3db1f53\") failed: Property not found. retry in 714.805206ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:19.681230337Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_902ea001-1913-4130-9979-87a8d3db1f53","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b2f11b30-44fe-4b76-98c0-1891b1b0971a\") failed: Property not found. retry in 829.918608ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:19.904329696Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_b2f11b30-44fe-4b76-98c0-1891b1b0971a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_eed4ad2f-7765-43e1-8943-ea458ed22e57\") failed: Property not found. retry in 969.542375ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:20.220508135Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_eed4ad2f-7765-43e1-8943-ea458ed22e57","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_acae5a41-5da9-420f-9bc8-154e4abdaf87\") failed: Property not found. retry in 926.738786ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:20.290690557Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_acae5a41-5da9-420f-9bc8-154e4abdaf87","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_902ea001-1913-4130-9979-87a8d3db1f53\") failed: Property not found. retry in 968.186512ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:20.407517431Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_902ea001-1913-4130-9979-87a8d3db1f53","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3c470f26-8422-4139-9f7f-81a0d84becba\") failed: Property not found. retry in 1.090332609s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:20.711499245Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_3c470f26-8422-4139-9f7f-81a0d84becba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b2f11b30-44fe-4b76-98c0-1891b1b0971a\") failed: Property not found. retry in 649.388012ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:20.746207095Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_b2f11b30-44fe-4b76-98c0-1891b1b0971a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_eed4ad2f-7765-43e1-8943-ea458ed22e57\") failed: Property not found. retry in 941.901562ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:21.207029947Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_eed4ad2f-7765-43e1-8943-ea458ed22e57","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_acae5a41-5da9-420f-9bc8-154e4abdaf87\") failed: Property not found. retry in 926.465875ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:21.232974219Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_acae5a41-5da9-420f-9bc8-154e4abdaf87","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_902ea001-1913-4130-9979-87a8d3db1f53\") failed: Property not found. retry in 547.68534ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:21.39340709Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_902ea001-1913-4130-9979-87a8d3db1f53","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b2f11b30-44fe-4b76-98c0-1891b1b0971a\") failed: Property not found. retry in 608.308828ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:21.409178078Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_b2f11b30-44fe-4b76-98c0-1891b1b0971a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3c470f26-8422-4139-9f7f-81a0d84becba\") failed: Property not found. retry in 1.159070816s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:21.816931784Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_3c470f26-8422-4139-9f7f-81a0d84becba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_902ea001-1913-4130-9979-87a8d3db1f53\") failed: Property not found. retry in 824.908414ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:21.951109626Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_902ea001-1913-4130-9979-87a8d3db1f53","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b2f11b30-44fe-4b76-98c0-1891b1b0971a\") failed: Property not found. retry in 1.105285073s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:22.027402828Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_b2f11b30-44fe-4b76-98c0-1891b1b0971a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_eed4ad2f-7765-43e1-8943-ea458ed22e57\") failed: Property not found. retry in 501.622866ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:22.158122941Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_eed4ad2f-7765-43e1-8943-ea458ed22e57","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_acae5a41-5da9-420f-9bc8-154e4abdaf87\") failed: Property not found. retry in 1.432271208s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:22.170723658Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_acae5a41-5da9-420f-9bc8-154e4abdaf87","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_eed4ad2f-7765-43e1-8943-ea458ed22e57\") failed: Property not found. retry in 974.119002ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:22.667984984Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_eed4ad2f-7765-43e1-8943-ea458ed22e57","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_902ea001-1913-4130-9979-87a8d3db1f53\") failed: Property not found. retry in 1.19824924s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:22.787593969Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_902ea001-1913-4130-9979-87a8d3db1f53","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3c470f26-8422-4139-9f7f-81a0d84becba\") failed: Property not found. retry in 766.116091ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:22.987132218Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_3c470f26-8422-4139-9f7f-81a0d84becba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b2f11b30-44fe-4b76-98c0-1891b1b0971a\") failed: Property not found. retry in 869.5578ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:23.144440653Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_b2f11b30-44fe-4b76-98c0-1891b1b0971a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:23.193930482Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:65191"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:23.194111367Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"938e0ca9-7f8a-43c6-ac54-a5fc176a7e84","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:65191","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":88,"response_code":"200"} {"msg":"wait for service Some(\"oxz_cockroachdb_acae5a41-5da9-420f-9bc8-154e4abdaf87\") failed: Property not found. retry in 1.206309915s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:23.616973515Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_acae5a41-5da9-420f-9bc8-154e4abdaf87","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_eed4ad2f-7765-43e1-8943-ea458ed22e57\") failed: Property not found. retry in 975.655348ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:23.654086442Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_eed4ad2f-7765-43e1-8943-ea458ed22e57","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_3c470f26-8422-4139-9f7f-81a0d84becba\") failed: Property not found. retry in 977.411288ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:23.764117021Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_3c470f26-8422-4139-9f7f-81a0d84becba","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_902ea001-1913-4130-9979-87a8d3db1f53\") failed: Property not found. retry in 1.462498023s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:23.998390175Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_902ea001-1913-4130-9979-87a8d3db1f53","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_b2f11b30-44fe-4b76-98c0-1891b1b0971a\") failed: Property not found. retry in 626.07511ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:24.026343413Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_b2f11b30-44fe-4b76-98c0-1891b1b0971a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_eed4ad2f-7765-43e1-8943-ea458ed22e57\") failed: Property not found. retry in 1.077199798s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:24.64351088Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_eed4ad2f-7765-43e1-8943-ea458ed22e57","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_eed4ad2f-7765-43e1-8943-ea458ed22e57\") failed: Property not found. retry in 1.421728126s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:25.734308113Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_eed4ad2f-7765-43e1-8943-ea458ed22e57","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_eed4ad2f-7765-43e1-8943-ea458ed22e57\") failed: Property not found. retry in 1.352387763s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:27.171354866Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_eed4ad2f-7765-43e1-8943-ea458ed22e57","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_eed4ad2f-7765-43e1-8943-ea458ed22e57\") failed: Property not found. retry in 1.244689885s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:28.542307144Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_eed4ad2f-7765-43e1-8943-ea458ed22e57","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:29.065626938Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","component":"MetricsManager","sled_id":"3abf3a4f-8820-40af-a01f-4369b42a5529","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"461.754385ms"} {"msg":"wait for service Some(\"oxz_cockroachdb_eed4ad2f-7765-43e1-8943-ea458ed22e57\") failed: Property not found. retry in 713.411936ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:29.799765837Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_eed4ad2f-7765-43e1-8943-ea458ed22e57","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_eed4ad2f-7765-43e1-8943-ea458ed22e57\") failed: Property not found. retry in 1.480677947s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:30.524437929Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_eed4ad2f-7765-43e1-8943-ea458ed22e57","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_eed4ad2f-7765-43e1-8943-ea458ed22e57\") failed: Property not found. retry in 1.087141234s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:32.01882452Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_eed4ad2f-7765-43e1-8943-ea458ed22e57","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_eed4ad2f-7765-43e1-8943-ea458ed22e57\") failed: Property not found. retry in 787.566197ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:33.118551801Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_eed4ad2f-7765-43e1-8943-ea458ed22e57","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_cockroachdb_eed4ad2f-7765-43e1-8943-ea458ed22e57\") failed: Property not found. retry in 1.181913108s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:33.917578389Z","hostname":"buskin","pid":1555,"zone":"oxz_cockroachdb_eed4ad2f-7765-43e1-8943-ea458ed22e57","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:35.127363521Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3061","zone":"oxz_cockroachdb_3c470f26-8422-4139-9f7f-81a0d84becba"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:35.127437211Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3061","zone":"oxz_cockroachdb_902ea001-1913-4130-9979-87a8d3db1f53"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:35.127451948Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3061","zone":"oxz_cockroachdb_acae5a41-5da9-420f-9bc8-154e4abdaf87"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:35.127458445Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3061","zone":"oxz_cockroachdb_eed4ad2f-7765-43e1-8943-ea458ed22e57"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:35.127463105Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3061","zone":"oxz_cockroachdb_b2f11b30-44fe-4b76-98c0-1891b1b0971a"} {"msg":"Writing ledger to /pool/int/4576fedf-479b-43b3-8fad-bb8b7ec274f8/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:35.127467895Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/d09b3f64-0a36-4052-8233-a94e62e662e2/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:35.12876801Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:35.129662324Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"351cf85d-c913-4a8c-b8b6-7d60b0d7120b","remote_addr":"[fd00:1122:3344:101::1]:55017","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":24182462,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:35.164904928Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:39257"} {"msg":"Initializing CRDB Cluster - sending request to [fd00:1122:3344:101::3]:32221","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:35.203288417Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3484"} {"msg":"Formatting CRDB","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:35.461890746Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3502"} {"msg":"Formatting CRDB - Completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:49.996031404Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3525"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:49.996143605Z","hostname":"buskin","pid":1555,"uri":"/cockroachdb","method":"POST","req_id":"a698c01f-9ce1-4c43-836d-ecb834147450","remote_addr":"[fd00:1122:3344:101::1]:39257","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":14790851,"response_code":"204"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:50.000620651Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","component":"MetricsManager","sled_id":"3abf3a4f-8820-40af-a01f-4369b42a5529","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"507.291197ms"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 83a194e2-3464-4a84-ae47-e33a0983bc56, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: fe17df80-010a-4b68-9ebc-6a94437be875, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5b6cee62-004c-4843-9e16-c25d2b0c743b, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6300d678-c957-4259-9a3e-c167ee4d0fb0, kind: Service { id: 5b6cee62-004c-4843-9e16-c25d2b0c743b }, name: Name(\"ntp-5b6cee62-004c-4843-9e16-c25d2b0c743b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 71])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 3c470f26-8422-4139-9f7f-81a0d84becba, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: acae5a41-5da9-420f-9bc8-154e4abdaf87, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } } } }, OmicronZoneConfig { id: eed4ad2f-7765-43e1-8943-ea458ed22e57, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } } } }, OmicronZoneConfig { id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 902ea001-1913-4130-9979-87a8d3db1f53, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8995285b-5cb0-4d3a-b021-8aec05314a65, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: c8555f5b-90c8-4b5b-b73e-5b0ba7ce0695, kind: Service { id: 8995285b-5cb0-4d3a-b021-8aec05314a65 }, name: Name(\"external-dns-8995285b-5cb0-4d3a-b021-8aec05314a65\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 233, 199])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 0a07869d-c9ec-4ade-b114-97ceff54b60e, kind: Service { id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8 }, name: Name(\"external-dns-8a344ff3-4023-4f5e-a29a-3db2dc6980b8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 206, 204])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 866d6328-4ca1-435e-8a98-cd5268df29e8, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 6eedd1d7-7b63-4139-96c3-ad58e9233e7c, kind: Service { id: 866d6328-4ca1-435e-8a98-cd5268df29e8 }, name: Name(\"nexus-866d6328-4ca1-435e-8a98-cd5268df29e8\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 250])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4952ec5d-23ba-48ea-9509-011fcb02adaf, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 38978e97-2109-4173-8624-98de2b729958, kind: Service { id: 4952ec5d-23ba-48ea-9509-011fcb02adaf }, name: Name(\"nexus-4952ec5d-23ba-48ea-9509-011fcb02adaf\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 188])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1c82ddd6-6f5a-4a92-a9b3-fde01c046c42, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 51730dfc-654f-49d5-a0b7-5f1b747fb3be, kind: Service { id: 1c82ddd6-6f5a-4a92-a9b3-fde01c046c42 }, name: Name(\"nexus-1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 28])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 893afdd2-39c6-4f94-b381-facb1a40bbb1, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 061f99f8-6626-4a81-8262-abd77ceeb36a, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 15711896-d3e4-44e3-b330-624f0f5e064d, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: c02f008f-19c7-48c5-93c6-3f776c51e015, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 738351b7-dacb-486a-ab38-cf8aefaef9cb, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 29db7ccb-90e1-414b-aeee-a9393f639631, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } } } }, OmicronZoneConfig { id: d857801a-a95b-4937-a2f0-31ad7c239ac7, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External } } } }, OmicronZoneConfig { id: 58b140e5-f61e-4410-8851-af929dfd8069, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External } } } }, OmicronZoneConfig { id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External } } } }, OmicronZoneConfig { id: c1741444-7ab7-4888-aa3c-e7b6da3527da, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:50.031134056Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:50.031251133Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::1]:59148"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83a194e2-3464-4a84-ae47-e33a0983bc56, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:50.031626524Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:50.031633611Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 83a194e2-3464-4a84-ae47-e33a0983bc56, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:50.054079497Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:50.0542015Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe17df80-010a-4b68-9ebc-6a94437be875, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:50.074981208Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:50.075033662Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: fe17df80-010a-4b68-9ebc-6a94437be875, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c470f26-8422-4139-9f7f-81a0d84becba, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:50.097144914Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:50.097186728Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 3c470f26-8422-4139-9f7f-81a0d84becba, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: acae5a41-5da9-420f-9bc8-154e4abdaf87, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:50.116040055Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:50.116078652Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: acae5a41-5da9-420f-9bc8-154e4abdaf87, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eed4ad2f-7765-43e1-8943-ea458ed22e57, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:50.135533437Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:50.135577923Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: eed4ad2f-7765-43e1-8943-ea458ed22e57, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:50.153758446Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:50.153804036Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 902ea001-1913-4130-9979-87a8d3db1f53, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:50.173895206Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:50.173939842Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 902ea001-1913-4130-9979-87a8d3db1f53, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8995285b-5cb0-4d3a-b021-8aec05314a65, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:50.194246667Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:50.194290252Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8995285b-5cb0-4d3a-b021-8aec05314a65, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:50.343262908Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:50.343316627Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 061f99f8-6626-4a81-8262-abd77ceeb36a, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:50.474383994Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:50.474432378Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 061f99f8-6626-4a81-8262-abd77ceeb36a, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 29db7ccb-90e1-414b-aeee-a9393f639631, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:50.608352401Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:50.608412843Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 29db7ccb-90e1-414b-aeee-a9393f639631, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:50.790787337Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:50.790835704Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d857801a-a95b-4937-a2f0-31ad7c239ac7, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:50.938725714Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:50.938778093Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: d857801a-a95b-4937-a2f0-31ad7c239ac7, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:51.058357062Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:51.058411541Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:51.184136883Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:51.184254009Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 58b140e5-f61e-4410-8851-af929dfd8069, dataset_name: DatasetName { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:51.30335931Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:51.303412645Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 58b140e5-f61e-4410-8851-af929dfd8069, dataset_name: DatasetName { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:51.436152629Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:51.436206583Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, dataset_name: DatasetName { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:51.579163396Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:51.579210788Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, dataset_name: DatasetName { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1741444-7ab7-4888-aa3c-e7b6da3527da, dataset_name: DatasetName { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:51.7383143Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:51.738370138Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: c1741444-7ab7-4888-aa3c-e7b6da3527da, dataset_name: DatasetName { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:51.878520481Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Reading ledger from /pool/int/4576fedf-479b-43b3-8fad-bb8b7ec274f8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:51.878595322Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/d09b3f64-0a36-4052-8233-a94e62e662e2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:51.878900747Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:51.879139574Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:52.000102538Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:52.000151186Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:52.000158058Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:52.000163917Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:52.000169402Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:52.000176471Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:52.000180558Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:52.000186071Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:52.000191715Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:52.00019884Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:52.000204606Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:52.000211806Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:52.000217375Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:52.000222596Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:52.000236603Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:52.000243607Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:52.000249026Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:52.000252551Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:52.000256376Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:52.000342251Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"1153b55a-081c-4b24-ab56-db62e8a5a6b4","remote_addr":"[fd00:1122:3344:101::1]:59148","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8\", TimeNotSynchronized), (\"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300\", TimeNotSynchronized), (\"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb\", TimeNotSynchronized), (\"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d\", TimeNotSynchronized), (\"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069\", TimeNotSynchronized), (\"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2\", TimeNotSynchronized), (\"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da\", TimeNotSynchronized), (\"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\", TimeNotSynchronized), (\"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451\", TimeNotSynchronized), (\"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8\", TimeNotSynchronized), (\"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65\", TimeNotSynchronized), (\"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1\", TimeNotSynchronized), (\"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d\", TimeNotSynchronized), (\"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf\", TimeNotSynchronized), (\"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015\", TimeNotSynchronized), (\"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a\", TimeNotSynchronized), (\"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631\", TimeNotSynchronized), (\"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83\", TimeNotSynchronized), (\"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7\", TimeNotSynchronized)]","latency_us":1963668,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:52.000600617Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"131.947908ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"1153b55a-081c-4b24-ab56-db62e8a5a6b4\", \"content-length\": \"124\", \"date\": \"Mon, 05 Aug 2024 16:43:52 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"1153b55a-081c-4b24-ab56-db62e8a5a6b4\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 83a194e2-3464-4a84-ae47-e33a0983bc56, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: fe17df80-010a-4b68-9ebc-6a94437be875, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5b6cee62-004c-4843-9e16-c25d2b0c743b, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6300d678-c957-4259-9a3e-c167ee4d0fb0, kind: Service { id: 5b6cee62-004c-4843-9e16-c25d2b0c743b }, name: Name(\"ntp-5b6cee62-004c-4843-9e16-c25d2b0c743b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 71])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 3c470f26-8422-4139-9f7f-81a0d84becba, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: acae5a41-5da9-420f-9bc8-154e4abdaf87, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } } } }, OmicronZoneConfig { id: eed4ad2f-7765-43e1-8943-ea458ed22e57, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } } } }, OmicronZoneConfig { id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 902ea001-1913-4130-9979-87a8d3db1f53, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8995285b-5cb0-4d3a-b021-8aec05314a65, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: c8555f5b-90c8-4b5b-b73e-5b0ba7ce0695, kind: Service { id: 8995285b-5cb0-4d3a-b021-8aec05314a65 }, name: Name(\"external-dns-8995285b-5cb0-4d3a-b021-8aec05314a65\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 233, 199])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 0a07869d-c9ec-4ade-b114-97ceff54b60e, kind: Service { id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8 }, name: Name(\"external-dns-8a344ff3-4023-4f5e-a29a-3db2dc6980b8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 206, 204])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 866d6328-4ca1-435e-8a98-cd5268df29e8, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 6eedd1d7-7b63-4139-96c3-ad58e9233e7c, kind: Service { id: 866d6328-4ca1-435e-8a98-cd5268df29e8 }, name: Name(\"nexus-866d6328-4ca1-435e-8a98-cd5268df29e8\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 250])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4952ec5d-23ba-48ea-9509-011fcb02adaf, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 38978e97-2109-4173-8624-98de2b729958, kind: Service { id: 4952ec5d-23ba-48ea-9509-011fcb02adaf }, name: Name(\"nexus-4952ec5d-23ba-48ea-9509-011fcb02adaf\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 188])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1c82ddd6-6f5a-4a92-a9b3-fde01c046c42, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 51730dfc-654f-49d5-a0b7-5f1b747fb3be, kind: Service { id: 1c82ddd6-6f5a-4a92-a9b3-fde01c046c42 }, name: Name(\"nexus-1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 28])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 893afdd2-39c6-4f94-b381-facb1a40bbb1, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 061f99f8-6626-4a81-8262-abd77ceeb36a, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 15711896-d3e4-44e3-b330-624f0f5e064d, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: c02f008f-19c7-48c5-93c6-3f776c51e015, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 738351b7-dacb-486a-ab38-cf8aefaef9cb, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 29db7ccb-90e1-414b-aeee-a9393f639631, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } } } }, OmicronZoneConfig { id: d857801a-a95b-4937-a2f0-31ad7c239ac7, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External } } } }, OmicronZoneConfig { id: 58b140e5-f61e-4410-8851-af929dfd8069, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External } } } }, OmicronZoneConfig { id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External } } } }, OmicronZoneConfig { id: c1741444-7ab7-4888-aa3c-e7b6da3527da, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:52.133215136Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83a194e2-3464-4a84-ae47-e33a0983bc56, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:52.133539225Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:52.133549764Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 83a194e2-3464-4a84-ae47-e33a0983bc56, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:52.155670285Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:52.155719205Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe17df80-010a-4b68-9ebc-6a94437be875, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:52.177697032Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:52.1777371Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: fe17df80-010a-4b68-9ebc-6a94437be875, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c470f26-8422-4139-9f7f-81a0d84becba, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:52.197258687Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:52.197302356Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 3c470f26-8422-4139-9f7f-81a0d84becba, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: acae5a41-5da9-420f-9bc8-154e4abdaf87, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:52.21652969Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:52.216572348Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: acae5a41-5da9-420f-9bc8-154e4abdaf87, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eed4ad2f-7765-43e1-8943-ea458ed22e57, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:52.23426793Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:52.234303948Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: eed4ad2f-7765-43e1-8943-ea458ed22e57, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:52.253022291Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:52.253132292Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 902ea001-1913-4130-9979-87a8d3db1f53, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:52.271763479Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:52.271809754Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 902ea001-1913-4130-9979-87a8d3db1f53, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8995285b-5cb0-4d3a-b021-8aec05314a65, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:52.290540413Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:52.290583978Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8995285b-5cb0-4d3a-b021-8aec05314a65, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:52.309711053Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:52.309753635Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 061f99f8-6626-4a81-8262-abd77ceeb36a, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:52.330881872Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:52.330920686Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 061f99f8-6626-4a81-8262-abd77ceeb36a, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 29db7ccb-90e1-414b-aeee-a9393f639631, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:52.349593732Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:52.349632198Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 29db7ccb-90e1-414b-aeee-a9393f639631, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:52.369498024Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:52.369549035Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d857801a-a95b-4937-a2f0-31ad7c239ac7, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:52.389939276Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:52.389977762Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: d857801a-a95b-4937-a2f0-31ad7c239ac7, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:52.409260436Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:52.409304012Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:52.428521078Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:52.42862904Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 58b140e5-f61e-4410-8851-af929dfd8069, dataset_name: DatasetName { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:52.447039815Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:52.447082481Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 58b140e5-f61e-4410-8851-af929dfd8069, dataset_name: DatasetName { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:52.465086006Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:52.465133172Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, dataset_name: DatasetName { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:52.484684477Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:52.484789309Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, dataset_name: DatasetName { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1741444-7ab7-4888-aa3c-e7b6da3527da, dataset_name: DatasetName { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:52.505150796Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:52.505193275Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: c1741444-7ab7-4888-aa3c-e7b6da3527da, dataset_name: DatasetName { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:52.524387258Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Reading ledger from /pool/int/4576fedf-479b-43b3-8fad-bb8b7ec274f8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:52.52443134Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/d09b3f64-0a36-4052-8233-a94e62e662e2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:52.524639132Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:52.524773534Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:52.623179242Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:52.623239374Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:52.623267827Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:52.623275478Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:52.623280878Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:52.623287938Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:52.623294669Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:52.623300275Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:52.623306697Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:52.623313166Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:52.623319597Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:52.623324641Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:52.623330838Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:52.623334635Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:52.623339398Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:52.62334296Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:52.62334792Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:52.623356967Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:52.623362563Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:52.623426024Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"964ac0cc-7b55-4fdd-b25e-82c932a006d0","remote_addr":"[fd00:1122:3344:101::1]:59148","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8\", TimeNotSynchronized), (\"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83\", TimeNotSynchronized), (\"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a\", TimeNotSynchronized), (\"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8\", TimeNotSynchronized), (\"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015\", TimeNotSynchronized), (\"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451\", TimeNotSynchronized), (\"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2\", TimeNotSynchronized), (\"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da\", TimeNotSynchronized), (\"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf\", TimeNotSynchronized), (\"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65\", TimeNotSynchronized), (\"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d\", TimeNotSynchronized), (\"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\", TimeNotSynchronized), (\"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d\", TimeNotSynchronized), (\"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1\", TimeNotSynchronized), (\"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631\", TimeNotSynchronized), (\"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7\", TimeNotSynchronized), (\"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069\", TimeNotSynchronized), (\"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb\", TimeNotSynchronized), (\"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300\", TimeNotSynchronized)]","latency_us":488719,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:52.623585823Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"151.60857ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"964ac0cc-7b55-4fdd-b25e-82c932a006d0\", \"content-length\": \"124\", \"date\": \"Mon, 05 Aug 2024 16:43:52 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"964ac0cc-7b55-4fdd-b25e-82c932a006d0\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 83a194e2-3464-4a84-ae47-e33a0983bc56, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: fe17df80-010a-4b68-9ebc-6a94437be875, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5b6cee62-004c-4843-9e16-c25d2b0c743b, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6300d678-c957-4259-9a3e-c167ee4d0fb0, kind: Service { id: 5b6cee62-004c-4843-9e16-c25d2b0c743b }, name: Name(\"ntp-5b6cee62-004c-4843-9e16-c25d2b0c743b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 71])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 3c470f26-8422-4139-9f7f-81a0d84becba, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: acae5a41-5da9-420f-9bc8-154e4abdaf87, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } } } }, OmicronZoneConfig { id: eed4ad2f-7765-43e1-8943-ea458ed22e57, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } } } }, OmicronZoneConfig { id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 902ea001-1913-4130-9979-87a8d3db1f53, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8995285b-5cb0-4d3a-b021-8aec05314a65, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: c8555f5b-90c8-4b5b-b73e-5b0ba7ce0695, kind: Service { id: 8995285b-5cb0-4d3a-b021-8aec05314a65 }, name: Name(\"external-dns-8995285b-5cb0-4d3a-b021-8aec05314a65\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 233, 199])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 0a07869d-c9ec-4ade-b114-97ceff54b60e, kind: Service { id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8 }, name: Name(\"external-dns-8a344ff3-4023-4f5e-a29a-3db2dc6980b8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 206, 204])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 866d6328-4ca1-435e-8a98-cd5268df29e8, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 6eedd1d7-7b63-4139-96c3-ad58e9233e7c, kind: Service { id: 866d6328-4ca1-435e-8a98-cd5268df29e8 }, name: Name(\"nexus-866d6328-4ca1-435e-8a98-cd5268df29e8\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 250])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4952ec5d-23ba-48ea-9509-011fcb02adaf, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 38978e97-2109-4173-8624-98de2b729958, kind: Service { id: 4952ec5d-23ba-48ea-9509-011fcb02adaf }, name: Name(\"nexus-4952ec5d-23ba-48ea-9509-011fcb02adaf\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 188])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1c82ddd6-6f5a-4a92-a9b3-fde01c046c42, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 51730dfc-654f-49d5-a0b7-5f1b747fb3be, kind: Service { id: 1c82ddd6-6f5a-4a92-a9b3-fde01c046c42 }, name: Name(\"nexus-1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 28])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 893afdd2-39c6-4f94-b381-facb1a40bbb1, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 061f99f8-6626-4a81-8262-abd77ceeb36a, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 15711896-d3e4-44e3-b330-624f0f5e064d, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: c02f008f-19c7-48c5-93c6-3f776c51e015, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 738351b7-dacb-486a-ab38-cf8aefaef9cb, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 29db7ccb-90e1-414b-aeee-a9393f639631, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } } } }, OmicronZoneConfig { id: d857801a-a95b-4937-a2f0-31ad7c239ac7, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External } } } }, OmicronZoneConfig { id: 58b140e5-f61e-4410-8851-af929dfd8069, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External } } } }, OmicronZoneConfig { id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External } } } }, OmicronZoneConfig { id: c1741444-7ab7-4888-aa3c-e7b6da3527da, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:52.77646236Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83a194e2-3464-4a84-ae47-e33a0983bc56, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:52.777347523Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:52.777355352Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 83a194e2-3464-4a84-ae47-e33a0983bc56, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:52.799487922Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:52.799530964Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe17df80-010a-4b68-9ebc-6a94437be875, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:52.819010058Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:52.819055201Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: fe17df80-010a-4b68-9ebc-6a94437be875, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c470f26-8422-4139-9f7f-81a0d84becba, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:52.838861692Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:52.838900861Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 3c470f26-8422-4139-9f7f-81a0d84becba, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: acae5a41-5da9-420f-9bc8-154e4abdaf87, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:52.856671815Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:52.856785731Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: acae5a41-5da9-420f-9bc8-154e4abdaf87, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eed4ad2f-7765-43e1-8943-ea458ed22e57, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:52.879007071Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:52.879046699Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: eed4ad2f-7765-43e1-8943-ea458ed22e57, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:52.898355293Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:52.898393759Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 902ea001-1913-4130-9979-87a8d3db1f53, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:52.919453405Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:52.919496597Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 902ea001-1913-4130-9979-87a8d3db1f53, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8995285b-5cb0-4d3a-b021-8aec05314a65, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:52.938338801Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:52.938379302Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8995285b-5cb0-4d3a-b021-8aec05314a65, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:52.956607189Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:52.956642608Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 061f99f8-6626-4a81-8262-abd77ceeb36a, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:52.977286311Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:52.977322958Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 061f99f8-6626-4a81-8262-abd77ceeb36a, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 29db7ccb-90e1-414b-aeee-a9393f639631, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:52.997587543Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:52.997622483Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 29db7ccb-90e1-414b-aeee-a9393f639631, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:53.016566083Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:53.016605926Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d857801a-a95b-4937-a2f0-31ad7c239ac7, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:53.03655162Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:53.036602874Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: d857801a-a95b-4937-a2f0-31ad7c239ac7, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:53.055012007Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:53.055056314Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:53.073867592Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:53.073903987Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 58b140e5-f61e-4410-8851-af929dfd8069, dataset_name: DatasetName { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:53.091809624Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:53.091849131Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 58b140e5-f61e-4410-8851-af929dfd8069, dataset_name: DatasetName { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:53.111844864Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:53.111889274Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, dataset_name: DatasetName { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:53.131653539Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:53.13170318Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, dataset_name: DatasetName { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1741444-7ab7-4888-aa3c-e7b6da3527da, dataset_name: DatasetName { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:53.151437641Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:53.151480804Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: c1741444-7ab7-4888-aa3c-e7b6da3527da, dataset_name: DatasetName { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:53.17209349Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Reading ledger from /pool/int/4576fedf-479b-43b3-8fad-bb8b7ec274f8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:53.172145128Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/d09b3f64-0a36-4052-8233-a94e62e662e2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:53.172323716Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:53.172438167Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:53.269514781Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:53.269566175Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:53.269572644Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:53.269576629Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:53.269580407Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:53.269585779Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:53.26961337Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:53.269620167Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:53.269625286Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:53.269631989Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:53.269637258Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:53.269644073Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:53.269649398Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:53.269655802Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:53.269662693Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:53.269666734Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:53.269670437Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:53.269673905Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:53.269678659Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:53.269756238Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"77424964-b71d-4a4b-b89e-cab15db65585","remote_addr":"[fd00:1122:3344:101::1]:59148","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d\", TimeNotSynchronized), (\"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015\", TimeNotSynchronized), (\"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631\", TimeNotSynchronized), (\"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83\", TimeNotSynchronized), (\"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8\", TimeNotSynchronized), (\"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2\", TimeNotSynchronized), (\"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d\", TimeNotSynchronized), (\"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8\", TimeNotSynchronized), (\"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf\", TimeNotSynchronized), (\"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\", TimeNotSynchronized), (\"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb\", TimeNotSynchronized), (\"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069\", TimeNotSynchronized), (\"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451\", TimeNotSynchronized), (\"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da\", TimeNotSynchronized), (\"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65\", TimeNotSynchronized), (\"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a\", TimeNotSynchronized), (\"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7\", TimeNotSynchronized), (\"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1\", TimeNotSynchronized), (\"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300\", TimeNotSynchronized)]","latency_us":491801,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:53.269977142Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"78.889799ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"77424964-b71d-4a4b-b89e-cab15db65585\", \"content-length\": \"124\", \"date\": \"Mon, 05 Aug 2024 16:43:53 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"77424964-b71d-4a4b-b89e-cab15db65585\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:53.309167701Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:47139"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:53.309238005Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"eba2179c-f6a8-469c-bf08-daea5a007c6c","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:47139","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":39,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 83a194e2-3464-4a84-ae47-e33a0983bc56, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: fe17df80-010a-4b68-9ebc-6a94437be875, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5b6cee62-004c-4843-9e16-c25d2b0c743b, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6300d678-c957-4259-9a3e-c167ee4d0fb0, kind: Service { id: 5b6cee62-004c-4843-9e16-c25d2b0c743b }, name: Name(\"ntp-5b6cee62-004c-4843-9e16-c25d2b0c743b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 71])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 3c470f26-8422-4139-9f7f-81a0d84becba, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: acae5a41-5da9-420f-9bc8-154e4abdaf87, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } } } }, OmicronZoneConfig { id: eed4ad2f-7765-43e1-8943-ea458ed22e57, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } } } }, OmicronZoneConfig { id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 902ea001-1913-4130-9979-87a8d3db1f53, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8995285b-5cb0-4d3a-b021-8aec05314a65, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: c8555f5b-90c8-4b5b-b73e-5b0ba7ce0695, kind: Service { id: 8995285b-5cb0-4d3a-b021-8aec05314a65 }, name: Name(\"external-dns-8995285b-5cb0-4d3a-b021-8aec05314a65\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 233, 199])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 0a07869d-c9ec-4ade-b114-97ceff54b60e, kind: Service { id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8 }, name: Name(\"external-dns-8a344ff3-4023-4f5e-a29a-3db2dc6980b8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 206, 204])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 866d6328-4ca1-435e-8a98-cd5268df29e8, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 6eedd1d7-7b63-4139-96c3-ad58e9233e7c, kind: Service { id: 866d6328-4ca1-435e-8a98-cd5268df29e8 }, name: Name(\"nexus-866d6328-4ca1-435e-8a98-cd5268df29e8\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 250])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4952ec5d-23ba-48ea-9509-011fcb02adaf, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 38978e97-2109-4173-8624-98de2b729958, kind: Service { id: 4952ec5d-23ba-48ea-9509-011fcb02adaf }, name: Name(\"nexus-4952ec5d-23ba-48ea-9509-011fcb02adaf\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 188])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1c82ddd6-6f5a-4a92-a9b3-fde01c046c42, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 51730dfc-654f-49d5-a0b7-5f1b747fb3be, kind: Service { id: 1c82ddd6-6f5a-4a92-a9b3-fde01c046c42 }, name: Name(\"nexus-1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 28])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 893afdd2-39c6-4f94-b381-facb1a40bbb1, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 061f99f8-6626-4a81-8262-abd77ceeb36a, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 15711896-d3e4-44e3-b330-624f0f5e064d, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: c02f008f-19c7-48c5-93c6-3f776c51e015, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 738351b7-dacb-486a-ab38-cf8aefaef9cb, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 29db7ccb-90e1-414b-aeee-a9393f639631, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } } } }, OmicronZoneConfig { id: d857801a-a95b-4937-a2f0-31ad7c239ac7, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External } } } }, OmicronZoneConfig { id: 58b140e5-f61e-4410-8851-af929dfd8069, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External } } } }, OmicronZoneConfig { id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External } } } }, OmicronZoneConfig { id: c1741444-7ab7-4888-aa3c-e7b6da3527da, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:53.349736679Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83a194e2-3464-4a84-ae47-e33a0983bc56, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:53.350008942Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:53.350019762Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 83a194e2-3464-4a84-ae47-e33a0983bc56, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:53.371181059Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:53.371221672Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe17df80-010a-4b68-9ebc-6a94437be875, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:53.391795034Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:53.391837682Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: fe17df80-010a-4b68-9ebc-6a94437be875, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c470f26-8422-4139-9f7f-81a0d84becba, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:53.410898928Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:53.410941734Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 3c470f26-8422-4139-9f7f-81a0d84becba, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: acae5a41-5da9-420f-9bc8-154e4abdaf87, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:53.433047571Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:53.433097729Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: acae5a41-5da9-420f-9bc8-154e4abdaf87, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eed4ad2f-7765-43e1-8943-ea458ed22e57, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:53.454612225Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:53.454652594Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: eed4ad2f-7765-43e1-8943-ea458ed22e57, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:53.475362842Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:53.475470145Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 902ea001-1913-4130-9979-87a8d3db1f53, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:53.49624032Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:53.496281881Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 902ea001-1913-4130-9979-87a8d3db1f53, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8995285b-5cb0-4d3a-b021-8aec05314a65, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:53.51615145Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:53.516187618Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8995285b-5cb0-4d3a-b021-8aec05314a65, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:53.535328777Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:53.535367675Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 061f99f8-6626-4a81-8262-abd77ceeb36a, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:53.556787281Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:53.556823197Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 061f99f8-6626-4a81-8262-abd77ceeb36a, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 29db7ccb-90e1-414b-aeee-a9393f639631, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:53.576718694Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:53.57675447Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 29db7ccb-90e1-414b-aeee-a9393f639631, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:53.594669127Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:53.594702558Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d857801a-a95b-4937-a2f0-31ad7c239ac7, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:53.615062586Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:53.615101848Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: d857801a-a95b-4937-a2f0-31ad7c239ac7, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:53.635527986Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:53.635565656Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:53.655568432Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:53.655612505Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 58b140e5-f61e-4410-8851-af929dfd8069, dataset_name: DatasetName { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:53.673886267Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:53.673993174Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 58b140e5-f61e-4410-8851-af929dfd8069, dataset_name: DatasetName { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:53.691696729Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:53.69173139Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, dataset_name: DatasetName { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:53.709724888Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:53.709773789Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, dataset_name: DatasetName { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1741444-7ab7-4888-aa3c-e7b6da3527da, dataset_name: DatasetName { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:53.727712119Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:53.727752366Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: c1741444-7ab7-4888-aa3c-e7b6da3527da, dataset_name: DatasetName { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:53.749721815Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Reading ledger from /pool/int/4576fedf-479b-43b3-8fad-bb8b7ec274f8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:53.7497696Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/d09b3f64-0a36-4052-8233-a94e62e662e2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:53.749929006Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:53.750030464Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:53.846555958Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:53.846596422Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:53.846602665Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:53.846610034Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:53.846616784Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:53.846621772Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:53.846628166Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:53.846653751Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:53.846660688Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:53.846664372Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:53.846669877Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:53.846673392Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:53.846678989Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:53.846684483Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:53.846691317Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:53.846696642Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:53.84670313Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:53.846708295Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:53.846713189Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:53.846768259Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"6ee9034d-ec5b-4f10-a903-a759d9ee1b3e","remote_addr":"[fd00:1122:3344:101::1]:59148","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631\", TimeNotSynchronized), (\"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83\", TimeNotSynchronized), (\"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2\", TimeNotSynchronized), (\"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\", TimeNotSynchronized), (\"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a\", TimeNotSynchronized), (\"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069\", TimeNotSynchronized), (\"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8\", TimeNotSynchronized), (\"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1\", TimeNotSynchronized), (\"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65\", TimeNotSynchronized), (\"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb\", TimeNotSynchronized), (\"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d\", TimeNotSynchronized), (\"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d\", TimeNotSynchronized), (\"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015\", TimeNotSynchronized), (\"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7\", TimeNotSynchronized), (\"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451\", TimeNotSynchronized), (\"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300\", TimeNotSynchronized), (\"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da\", TimeNotSynchronized), (\"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8\", TimeNotSynchronized), (\"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf\", TimeNotSynchronized)]","latency_us":495556,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:53.846934471Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"230.952503ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"6ee9034d-ec5b-4f10-a903-a759d9ee1b3e\", \"content-length\": \"124\", \"date\": \"Mon, 05 Aug 2024 16:43:53 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"6ee9034d-ec5b-4f10-a903-a759d9ee1b3e\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 83a194e2-3464-4a84-ae47-e33a0983bc56, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: fe17df80-010a-4b68-9ebc-6a94437be875, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5b6cee62-004c-4843-9e16-c25d2b0c743b, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6300d678-c957-4259-9a3e-c167ee4d0fb0, kind: Service { id: 5b6cee62-004c-4843-9e16-c25d2b0c743b }, name: Name(\"ntp-5b6cee62-004c-4843-9e16-c25d2b0c743b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 71])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 3c470f26-8422-4139-9f7f-81a0d84becba, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: acae5a41-5da9-420f-9bc8-154e4abdaf87, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } } } }, OmicronZoneConfig { id: eed4ad2f-7765-43e1-8943-ea458ed22e57, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } } } }, OmicronZoneConfig { id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 902ea001-1913-4130-9979-87a8d3db1f53, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8995285b-5cb0-4d3a-b021-8aec05314a65, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: c8555f5b-90c8-4b5b-b73e-5b0ba7ce0695, kind: Service { id: 8995285b-5cb0-4d3a-b021-8aec05314a65 }, name: Name(\"external-dns-8995285b-5cb0-4d3a-b021-8aec05314a65\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 233, 199])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 0a07869d-c9ec-4ade-b114-97ceff54b60e, kind: Service { id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8 }, name: Name(\"external-dns-8a344ff3-4023-4f5e-a29a-3db2dc6980b8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 206, 204])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 866d6328-4ca1-435e-8a98-cd5268df29e8, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 6eedd1d7-7b63-4139-96c3-ad58e9233e7c, kind: Service { id: 866d6328-4ca1-435e-8a98-cd5268df29e8 }, name: Name(\"nexus-866d6328-4ca1-435e-8a98-cd5268df29e8\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 250])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4952ec5d-23ba-48ea-9509-011fcb02adaf, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 38978e97-2109-4173-8624-98de2b729958, kind: Service { id: 4952ec5d-23ba-48ea-9509-011fcb02adaf }, name: Name(\"nexus-4952ec5d-23ba-48ea-9509-011fcb02adaf\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 188])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1c82ddd6-6f5a-4a92-a9b3-fde01c046c42, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 51730dfc-654f-49d5-a0b7-5f1b747fb3be, kind: Service { id: 1c82ddd6-6f5a-4a92-a9b3-fde01c046c42 }, name: Name(\"nexus-1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 28])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 893afdd2-39c6-4f94-b381-facb1a40bbb1, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 061f99f8-6626-4a81-8262-abd77ceeb36a, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 15711896-d3e4-44e3-b330-624f0f5e064d, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: c02f008f-19c7-48c5-93c6-3f776c51e015, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 738351b7-dacb-486a-ab38-cf8aefaef9cb, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 29db7ccb-90e1-414b-aeee-a9393f639631, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } } } }, OmicronZoneConfig { id: d857801a-a95b-4937-a2f0-31ad7c239ac7, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External } } } }, OmicronZoneConfig { id: 58b140e5-f61e-4410-8851-af929dfd8069, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External } } } }, OmicronZoneConfig { id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External } } } }, OmicronZoneConfig { id: c1741444-7ab7-4888-aa3c-e7b6da3527da, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:54.080005043Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83a194e2-3464-4a84-ae47-e33a0983bc56, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:54.080382694Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:54.080391976Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 83a194e2-3464-4a84-ae47-e33a0983bc56, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:54.102651931Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:54.102693595Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe17df80-010a-4b68-9ebc-6a94437be875, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:54.122876375Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:54.122912788Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: fe17df80-010a-4b68-9ebc-6a94437be875, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c470f26-8422-4139-9f7f-81a0d84becba, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:54.143363308Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:54.143402084Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 3c470f26-8422-4139-9f7f-81a0d84becba, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: acae5a41-5da9-420f-9bc8-154e4abdaf87, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:54.162910644Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:54.162946476Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: acae5a41-5da9-420f-9bc8-154e4abdaf87, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eed4ad2f-7765-43e1-8943-ea458ed22e57, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:54.183227945Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:54.183260234Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: eed4ad2f-7765-43e1-8943-ea458ed22e57, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:54.202090934Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:54.202129598Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 902ea001-1913-4130-9979-87a8d3db1f53, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:54.221821458Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:54.221862531Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 902ea001-1913-4130-9979-87a8d3db1f53, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8995285b-5cb0-4d3a-b021-8aec05314a65, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:54.240563593Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:54.24059589Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8995285b-5cb0-4d3a-b021-8aec05314a65, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:54.259673485Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:54.259708389Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 061f99f8-6626-4a81-8262-abd77ceeb36a, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:54.280098277Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:54.280137502Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 061f99f8-6626-4a81-8262-abd77ceeb36a, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 29db7ccb-90e1-414b-aeee-a9393f639631, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:54.300495055Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:54.300526818Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 29db7ccb-90e1-414b-aeee-a9393f639631, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:54.319447524Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:54.319478734Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d857801a-a95b-4937-a2f0-31ad7c239ac7, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:54.338944551Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:54.338977345Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: d857801a-a95b-4937-a2f0-31ad7c239ac7, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:54.358937746Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:54.358980131Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:54.378487811Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:54.378519828Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 58b140e5-f61e-4410-8851-af929dfd8069, dataset_name: DatasetName { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:54.398619936Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:54.398665996Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 58b140e5-f61e-4410-8851-af929dfd8069, dataset_name: DatasetName { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:54.417650652Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:54.417686353Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, dataset_name: DatasetName { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:54.437739518Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:54.43776891Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, dataset_name: DatasetName { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1741444-7ab7-4888-aa3c-e7b6da3527da, dataset_name: DatasetName { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:54.45785536Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:54.457887555Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: c1741444-7ab7-4888-aa3c-e7b6da3527da, dataset_name: DatasetName { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:54.476532509Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Reading ledger from /pool/int/4576fedf-479b-43b3-8fad-bb8b7ec274f8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:54.476577594Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/d09b3f64-0a36-4052-8233-a94e62e662e2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:54.476776207Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:54.476881172Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:54.571814367Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:54.57187478Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:54.571884136Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:54.571891074Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:54.571899174Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:54.571907724Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:54.571914156Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:54.571920259Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:54.571926662Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:54.571933075Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:54.571937556Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:54.571966347Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:54.57197306Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:54.571980626Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:54.571986954Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:54.571994838Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:54.572002957Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:54.572007701Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:54.572013467Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:54.572107715Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"34d9b261-bc03-4d9c-832e-73d7ec98651c","remote_addr":"[fd00:1122:3344:101::1]:59148","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d\", TimeNotSynchronized), (\"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8\", TimeNotSynchronized), (\"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65\", TimeNotSynchronized), (\"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf\", TimeNotSynchronized), (\"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451\", TimeNotSynchronized), (\"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8\", TimeNotSynchronized), (\"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1\", TimeNotSynchronized), (\"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb\", TimeNotSynchronized), (\"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da\", TimeNotSynchronized), (\"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069\", TimeNotSynchronized), (\"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83\", TimeNotSynchronized), (\"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2\", TimeNotSynchronized), (\"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015\", TimeNotSynchronized), (\"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300\", TimeNotSynchronized), (\"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631\", TimeNotSynchronized), (\"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\", TimeNotSynchronized), (\"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a\", TimeNotSynchronized), (\"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7\", TimeNotSynchronized), (\"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d\", TimeNotSynchronized)]","latency_us":490637,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:54.572273383Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"297.63195ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"34d9b261-bc03-4d9c-832e-73d7ec98651c\", \"content-length\": \"124\", \"date\": \"Mon, 05 Aug 2024 16:43:54 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"34d9b261-bc03-4d9c-832e-73d7ec98651c\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 83a194e2-3464-4a84-ae47-e33a0983bc56, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: fe17df80-010a-4b68-9ebc-6a94437be875, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5b6cee62-004c-4843-9e16-c25d2b0c743b, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6300d678-c957-4259-9a3e-c167ee4d0fb0, kind: Service { id: 5b6cee62-004c-4843-9e16-c25d2b0c743b }, name: Name(\"ntp-5b6cee62-004c-4843-9e16-c25d2b0c743b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 71])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 3c470f26-8422-4139-9f7f-81a0d84becba, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: acae5a41-5da9-420f-9bc8-154e4abdaf87, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } } } }, OmicronZoneConfig { id: eed4ad2f-7765-43e1-8943-ea458ed22e57, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } } } }, OmicronZoneConfig { id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 902ea001-1913-4130-9979-87a8d3db1f53, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8995285b-5cb0-4d3a-b021-8aec05314a65, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: c8555f5b-90c8-4b5b-b73e-5b0ba7ce0695, kind: Service { id: 8995285b-5cb0-4d3a-b021-8aec05314a65 }, name: Name(\"external-dns-8995285b-5cb0-4d3a-b021-8aec05314a65\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 233, 199])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 0a07869d-c9ec-4ade-b114-97ceff54b60e, kind: Service { id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8 }, name: Name(\"external-dns-8a344ff3-4023-4f5e-a29a-3db2dc6980b8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 206, 204])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 866d6328-4ca1-435e-8a98-cd5268df29e8, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 6eedd1d7-7b63-4139-96c3-ad58e9233e7c, kind: Service { id: 866d6328-4ca1-435e-8a98-cd5268df29e8 }, name: Name(\"nexus-866d6328-4ca1-435e-8a98-cd5268df29e8\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 250])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4952ec5d-23ba-48ea-9509-011fcb02adaf, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 38978e97-2109-4173-8624-98de2b729958, kind: Service { id: 4952ec5d-23ba-48ea-9509-011fcb02adaf }, name: Name(\"nexus-4952ec5d-23ba-48ea-9509-011fcb02adaf\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 188])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1c82ddd6-6f5a-4a92-a9b3-fde01c046c42, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 51730dfc-654f-49d5-a0b7-5f1b747fb3be, kind: Service { id: 1c82ddd6-6f5a-4a92-a9b3-fde01c046c42 }, name: Name(\"nexus-1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 28])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 893afdd2-39c6-4f94-b381-facb1a40bbb1, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 061f99f8-6626-4a81-8262-abd77ceeb36a, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 15711896-d3e4-44e3-b330-624f0f5e064d, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: c02f008f-19c7-48c5-93c6-3f776c51e015, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 738351b7-dacb-486a-ab38-cf8aefaef9cb, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 29db7ccb-90e1-414b-aeee-a9393f639631, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } } } }, OmicronZoneConfig { id: d857801a-a95b-4937-a2f0-31ad7c239ac7, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External } } } }, OmicronZoneConfig { id: 58b140e5-f61e-4410-8851-af929dfd8069, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External } } } }, OmicronZoneConfig { id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External } } } }, OmicronZoneConfig { id: c1741444-7ab7-4888-aa3c-e7b6da3527da, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:54.872453255Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83a194e2-3464-4a84-ae47-e33a0983bc56, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:54.872813832Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:54.872822468Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 83a194e2-3464-4a84-ae47-e33a0983bc56, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:54.896508287Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:54.896550588Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe17df80-010a-4b68-9ebc-6a94437be875, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:54.914358276Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:54.91440547Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: fe17df80-010a-4b68-9ebc-6a94437be875, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c470f26-8422-4139-9f7f-81a0d84becba, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:54.935842301Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:54.935882277Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 3c470f26-8422-4139-9f7f-81a0d84becba, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: acae5a41-5da9-420f-9bc8-154e4abdaf87, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:54.955750343Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:54.955786091Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: acae5a41-5da9-420f-9bc8-154e4abdaf87, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eed4ad2f-7765-43e1-8943-ea458ed22e57, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:54.972695557Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:54.972733835Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: eed4ad2f-7765-43e1-8943-ea458ed22e57, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:54.992241852Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:54.992284423Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 902ea001-1913-4130-9979-87a8d3db1f53, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:55.011305114Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:55.01135933Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 902ea001-1913-4130-9979-87a8d3db1f53, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8995285b-5cb0-4d3a-b021-8aec05314a65, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:55.032568123Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:55.032600785Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8995285b-5cb0-4d3a-b021-8aec05314a65, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:55.051855335Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:55.051894654Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 061f99f8-6626-4a81-8262-abd77ceeb36a, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:55.072268443Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:55.072309093Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 061f99f8-6626-4a81-8262-abd77ceeb36a, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 29db7ccb-90e1-414b-aeee-a9393f639631, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:55.092735543Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:55.092846937Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 29db7ccb-90e1-414b-aeee-a9393f639631, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:55.113146645Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:55.113186396Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d857801a-a95b-4937-a2f0-31ad7c239ac7, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:55.13251656Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:55.132558496Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: d857801a-a95b-4937-a2f0-31ad7c239ac7, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:55.152039282Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:55.152075076Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:55.170481935Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:55.170516566Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 58b140e5-f61e-4410-8851-af929dfd8069, dataset_name: DatasetName { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:55.191351091Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:55.191397366Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 58b140e5-f61e-4410-8851-af929dfd8069, dataset_name: DatasetName { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:55.209839292Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:55.209880234Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, dataset_name: DatasetName { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:55.228928445Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:55.228970905Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, dataset_name: DatasetName { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1741444-7ab7-4888-aa3c-e7b6da3527da, dataset_name: DatasetName { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:55.250029674Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:55.25006561Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: c1741444-7ab7-4888-aa3c-e7b6da3527da, dataset_name: DatasetName { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:55.269344207Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Reading ledger from /pool/int/4576fedf-479b-43b3-8fad-bb8b7ec274f8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:55.269393699Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/d09b3f64-0a36-4052-8233-a94e62e662e2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:55.269597214Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:55.269717113Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:55.364673935Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:55.364719743Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:55.364726605Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:55.364732418Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:55.364739618Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:55.364744793Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:55.364750222Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:55.364756775Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:55.364765484Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:55.364775328Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:55.364786165Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:55.364796365Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:55.364804166Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:55.36480996Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:55.364839726Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:55.364851145Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:55.364857099Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:55.36486458Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:55.364871892Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:55.36492235Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"a698c032-9f09-4711-9ed5-4b5c7aa799bb","remote_addr":"[fd00:1122:3344:101::1]:59148","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf\", TimeNotSynchronized), (\"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\", TimeNotSynchronized), (\"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d\", TimeNotSynchronized), (\"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015\", TimeNotSynchronized), (\"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb\", TimeNotSynchronized), (\"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631\", TimeNotSynchronized), (\"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8\", TimeNotSynchronized), (\"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a\", TimeNotSynchronized), (\"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83\", TimeNotSynchronized), (\"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d\", TimeNotSynchronized), (\"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069\", TimeNotSynchronized), (\"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451\", TimeNotSynchronized), (\"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65\", TimeNotSynchronized), (\"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da\", TimeNotSynchronized), (\"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8\", TimeNotSynchronized), (\"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7\", TimeNotSynchronized), (\"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2\", TimeNotSynchronized), (\"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300\", TimeNotSynchronized), (\"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1\", TimeNotSynchronized)]","latency_us":490986,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:55.365105868Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"229.168562ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a698c032-9f09-4711-9ed5-4b5c7aa799bb\", \"content-length\": \"124\", \"date\": \"Mon, 05 Aug 2024 16:43:55 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a698c032-9f09-4711-9ed5-4b5c7aa799bb\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 83a194e2-3464-4a84-ae47-e33a0983bc56, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: fe17df80-010a-4b68-9ebc-6a94437be875, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5b6cee62-004c-4843-9e16-c25d2b0c743b, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6300d678-c957-4259-9a3e-c167ee4d0fb0, kind: Service { id: 5b6cee62-004c-4843-9e16-c25d2b0c743b }, name: Name(\"ntp-5b6cee62-004c-4843-9e16-c25d2b0c743b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 71])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 3c470f26-8422-4139-9f7f-81a0d84becba, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: acae5a41-5da9-420f-9bc8-154e4abdaf87, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } } } }, OmicronZoneConfig { id: eed4ad2f-7765-43e1-8943-ea458ed22e57, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } } } }, OmicronZoneConfig { id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 902ea001-1913-4130-9979-87a8d3db1f53, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8995285b-5cb0-4d3a-b021-8aec05314a65, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: c8555f5b-90c8-4b5b-b73e-5b0ba7ce0695, kind: Service { id: 8995285b-5cb0-4d3a-b021-8aec05314a65 }, name: Name(\"external-dns-8995285b-5cb0-4d3a-b021-8aec05314a65\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 233, 199])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 0a07869d-c9ec-4ade-b114-97ceff54b60e, kind: Service { id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8 }, name: Name(\"external-dns-8a344ff3-4023-4f5e-a29a-3db2dc6980b8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 206, 204])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 866d6328-4ca1-435e-8a98-cd5268df29e8, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 6eedd1d7-7b63-4139-96c3-ad58e9233e7c, kind: Service { id: 866d6328-4ca1-435e-8a98-cd5268df29e8 }, name: Name(\"nexus-866d6328-4ca1-435e-8a98-cd5268df29e8\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 250])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4952ec5d-23ba-48ea-9509-011fcb02adaf, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 38978e97-2109-4173-8624-98de2b729958, kind: Service { id: 4952ec5d-23ba-48ea-9509-011fcb02adaf }, name: Name(\"nexus-4952ec5d-23ba-48ea-9509-011fcb02adaf\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 188])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1c82ddd6-6f5a-4a92-a9b3-fde01c046c42, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 51730dfc-654f-49d5-a0b7-5f1b747fb3be, kind: Service { id: 1c82ddd6-6f5a-4a92-a9b3-fde01c046c42 }, name: Name(\"nexus-1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 28])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 893afdd2-39c6-4f94-b381-facb1a40bbb1, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 061f99f8-6626-4a81-8262-abd77ceeb36a, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 15711896-d3e4-44e3-b330-624f0f5e064d, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: c02f008f-19c7-48c5-93c6-3f776c51e015, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 738351b7-dacb-486a-ab38-cf8aefaef9cb, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 29db7ccb-90e1-414b-aeee-a9393f639631, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } } } }, OmicronZoneConfig { id: d857801a-a95b-4937-a2f0-31ad7c239ac7, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External } } } }, OmicronZoneConfig { id: 58b140e5-f61e-4410-8851-af929dfd8069, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External } } } }, OmicronZoneConfig { id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External } } } }, OmicronZoneConfig { id: c1741444-7ab7-4888-aa3c-e7b6da3527da, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:55.596261646Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83a194e2-3464-4a84-ae47-e33a0983bc56, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:55.59728073Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:55.59728823Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 83a194e2-3464-4a84-ae47-e33a0983bc56, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:55.619773024Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:55.619834572Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe17df80-010a-4b68-9ebc-6a94437be875, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:55.638883391Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:55.638926489Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: fe17df80-010a-4b68-9ebc-6a94437be875, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c470f26-8422-4139-9f7f-81a0d84becba, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:55.659029332Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:55.659075683Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 3c470f26-8422-4139-9f7f-81a0d84becba, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: acae5a41-5da9-420f-9bc8-154e4abdaf87, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:55.678505844Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:55.678542603Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: acae5a41-5da9-420f-9bc8-154e4abdaf87, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eed4ad2f-7765-43e1-8943-ea458ed22e57, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:55.697236494Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:55.69727843Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: eed4ad2f-7765-43e1-8943-ea458ed22e57, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:55.717200741Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:55.717243136Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 902ea001-1913-4130-9979-87a8d3db1f53, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:55.736666092Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:55.736702495Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 902ea001-1913-4130-9979-87a8d3db1f53, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8995285b-5cb0-4d3a-b021-8aec05314a65, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:55.757396002Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:55.757436034Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8995285b-5cb0-4d3a-b021-8aec05314a65, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:55.776858516Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:55.776901098Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 061f99f8-6626-4a81-8262-abd77ceeb36a, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:55.797258947Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:55.797296373Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 061f99f8-6626-4a81-8262-abd77ceeb36a, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 29db7ccb-90e1-414b-aeee-a9393f639631, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:55.816577114Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:55.816618739Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 29db7ccb-90e1-414b-aeee-a9393f639631, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:55.835107634Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:55.835147882Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d857801a-a95b-4937-a2f0-31ad7c239ac7, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:55.853961892Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:55.853999879Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: d857801a-a95b-4937-a2f0-31ad7c239ac7, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:55.872295917Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:55.872334092Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:55.890563659Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:55.890607873Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 58b140e5-f61e-4410-8851-af929dfd8069, dataset_name: DatasetName { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:55.911744105Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:55.911781409Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 58b140e5-f61e-4410-8851-af929dfd8069, dataset_name: DatasetName { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:55.936048767Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:55.936169608Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, dataset_name: DatasetName { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:55.955806083Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:55.955852264Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, dataset_name: DatasetName { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1741444-7ab7-4888-aa3c-e7b6da3527da, dataset_name: DatasetName { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:55.974954865Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:55.97499125Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: c1741444-7ab7-4888-aa3c-e7b6da3527da, dataset_name: DatasetName { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:55.993369682Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Reading ledger from /pool/int/4576fedf-479b-43b3-8fad-bb8b7ec274f8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:55.993424114Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/d09b3f64-0a36-4052-8233-a94e62e662e2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:55.993596579Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:55.993706147Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:56.091767099Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:56.091817622Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:56.091827954Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:56.091844201Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:56.091850745Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:56.091857589Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:56.091863176Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:56.091868417Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:56.091875214Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:56.091880933Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:56.091886343Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:56.091893045Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:56.091901905Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:56.09190993Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:56.091917299Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:56.091926552Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:56.091936049Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:56.091944094Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:56.091968309Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:56.092115369Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"eeabf52b-d03b-4d4b-9511-52317510f7d8","remote_addr":"[fd00:1122:3344:101::1]:59148","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a\", TimeNotSynchronized), (\"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1\", TimeNotSynchronized), (\"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65\", TimeNotSynchronized), (\"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8\", TimeNotSynchronized), (\"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d\", TimeNotSynchronized), (\"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb\", TimeNotSynchronized), (\"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83\", TimeNotSynchronized), (\"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7\", TimeNotSynchronized), (\"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf\", TimeNotSynchronized), (\"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631\", TimeNotSynchronized), (\"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2\", TimeNotSynchronized), (\"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300\", TimeNotSynchronized), (\"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da\", TimeNotSynchronized), (\"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8\", TimeNotSynchronized), (\"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069\", TimeNotSynchronized), (\"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d\", TimeNotSynchronized), (\"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\", TimeNotSynchronized), (\"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451\", TimeNotSynchronized), (\"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015\", TimeNotSynchronized)]","latency_us":494402,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:56.092318509Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"417.393124ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"eeabf52b-d03b-4d4b-9511-52317510f7d8\", \"content-length\": \"124\", \"date\": \"Mon, 05 Aug 2024 16:43:56 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"eeabf52b-d03b-4d4b-9511-52317510f7d8\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 83a194e2-3464-4a84-ae47-e33a0983bc56, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: fe17df80-010a-4b68-9ebc-6a94437be875, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5b6cee62-004c-4843-9e16-c25d2b0c743b, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6300d678-c957-4259-9a3e-c167ee4d0fb0, kind: Service { id: 5b6cee62-004c-4843-9e16-c25d2b0c743b }, name: Name(\"ntp-5b6cee62-004c-4843-9e16-c25d2b0c743b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 71])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 3c470f26-8422-4139-9f7f-81a0d84becba, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: acae5a41-5da9-420f-9bc8-154e4abdaf87, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } } } }, OmicronZoneConfig { id: eed4ad2f-7765-43e1-8943-ea458ed22e57, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } } } }, OmicronZoneConfig { id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 902ea001-1913-4130-9979-87a8d3db1f53, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8995285b-5cb0-4d3a-b021-8aec05314a65, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: c8555f5b-90c8-4b5b-b73e-5b0ba7ce0695, kind: Service { id: 8995285b-5cb0-4d3a-b021-8aec05314a65 }, name: Name(\"external-dns-8995285b-5cb0-4d3a-b021-8aec05314a65\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 233, 199])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 0a07869d-c9ec-4ade-b114-97ceff54b60e, kind: Service { id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8 }, name: Name(\"external-dns-8a344ff3-4023-4f5e-a29a-3db2dc6980b8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 206, 204])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 866d6328-4ca1-435e-8a98-cd5268df29e8, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 6eedd1d7-7b63-4139-96c3-ad58e9233e7c, kind: Service { id: 866d6328-4ca1-435e-8a98-cd5268df29e8 }, name: Name(\"nexus-866d6328-4ca1-435e-8a98-cd5268df29e8\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 250])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4952ec5d-23ba-48ea-9509-011fcb02adaf, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 38978e97-2109-4173-8624-98de2b729958, kind: Service { id: 4952ec5d-23ba-48ea-9509-011fcb02adaf }, name: Name(\"nexus-4952ec5d-23ba-48ea-9509-011fcb02adaf\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 188])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1c82ddd6-6f5a-4a92-a9b3-fde01c046c42, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 51730dfc-654f-49d5-a0b7-5f1b747fb3be, kind: Service { id: 1c82ddd6-6f5a-4a92-a9b3-fde01c046c42 }, name: Name(\"nexus-1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 28])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 893afdd2-39c6-4f94-b381-facb1a40bbb1, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 061f99f8-6626-4a81-8262-abd77ceeb36a, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 15711896-d3e4-44e3-b330-624f0f5e064d, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: c02f008f-19c7-48c5-93c6-3f776c51e015, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 738351b7-dacb-486a-ab38-cf8aefaef9cb, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 29db7ccb-90e1-414b-aeee-a9393f639631, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } } } }, OmicronZoneConfig { id: d857801a-a95b-4937-a2f0-31ad7c239ac7, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External } } } }, OmicronZoneConfig { id: 58b140e5-f61e-4410-8851-af929dfd8069, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External } } } }, OmicronZoneConfig { id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External } } } }, OmicronZoneConfig { id: c1741444-7ab7-4888-aa3c-e7b6da3527da, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:56.512815282Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83a194e2-3464-4a84-ae47-e33a0983bc56, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:56.513121868Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:56.513128525Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 83a194e2-3464-4a84-ae47-e33a0983bc56, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:56.534197335Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:56.534242345Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe17df80-010a-4b68-9ebc-6a94437be875, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:56.554300349Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:56.554351078Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: fe17df80-010a-4b68-9ebc-6a94437be875, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c470f26-8422-4139-9f7f-81a0d84becba, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:56.577498521Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:56.577548378Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 3c470f26-8422-4139-9f7f-81a0d84becba, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: acae5a41-5da9-420f-9bc8-154e4abdaf87, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:56.598145078Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:56.598196182Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: acae5a41-5da9-420f-9bc8-154e4abdaf87, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eed4ad2f-7765-43e1-8943-ea458ed22e57, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:56.620620288Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:56.620666246Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: eed4ad2f-7765-43e1-8943-ea458ed22e57, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:56.639842966Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:56.639887114Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 902ea001-1913-4130-9979-87a8d3db1f53, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:56.659613456Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:56.659652241Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 902ea001-1913-4130-9979-87a8d3db1f53, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8995285b-5cb0-4d3a-b021-8aec05314a65, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:56.679778727Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:56.679823297Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8995285b-5cb0-4d3a-b021-8aec05314a65, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:56.700380454Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:56.700435111Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 061f99f8-6626-4a81-8262-abd77ceeb36a, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:56.719317579Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:56.719368084Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 061f99f8-6626-4a81-8262-abd77ceeb36a, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 29db7ccb-90e1-414b-aeee-a9393f639631, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:56.739450493Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:56.739488604Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 29db7ccb-90e1-414b-aeee-a9393f639631, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:56.76009813Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:56.76014374Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d857801a-a95b-4937-a2f0-31ad7c239ac7, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:56.779711536Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:56.779752937Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: d857801a-a95b-4937-a2f0-31ad7c239ac7, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:56.800110627Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:56.800152946Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:56.82128326Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:56.821329376Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 58b140e5-f61e-4410-8851-af929dfd8069, dataset_name: DatasetName { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:56.841993742Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:56.842032293Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 58b140e5-f61e-4410-8851-af929dfd8069, dataset_name: DatasetName { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:56.862909013Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:56.862954295Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, dataset_name: DatasetName { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:56.881390428Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:56.881454985Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, dataset_name: DatasetName { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1741444-7ab7-4888-aa3c-e7b6da3527da, dataset_name: DatasetName { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:56.902175577Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:56.902217493Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: c1741444-7ab7-4888-aa3c-e7b6da3527da, dataset_name: DatasetName { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:56.922235404Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Reading ledger from /pool/int/4576fedf-479b-43b3-8fad-bb8b7ec274f8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:56.922290877Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/d09b3f64-0a36-4052-8233-a94e62e662e2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:56.922481877Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:56.922604664Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:57.025277969Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:57.025338477Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:57.025345584Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:57.025352577Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:57.025357921Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:57.025363331Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:57.025370099Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:57.025388137Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:57.025394935Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:57.025401619Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:57.025406766Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:57.025413263Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:57.025418813Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:57.025424991Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:57.025428947Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:57.025434516Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:57.025438303Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:57.02544331Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:57.025448786Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:57.025519633Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"dc71e953-3718-4311-b0c4-d25a057af8e2","remote_addr":"[fd00:1122:3344:101::1]:59148","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\", TimeNotSynchronized), (\"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015\", TimeNotSynchronized), (\"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d\", TimeNotSynchronized), (\"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069\", TimeNotSynchronized), (\"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7\", TimeNotSynchronized), (\"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb\", TimeNotSynchronized), (\"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451\", TimeNotSynchronized), (\"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300\", TimeNotSynchronized), (\"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8\", TimeNotSynchronized), (\"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1\", TimeNotSynchronized), (\"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d\", TimeNotSynchronized), (\"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631\", TimeNotSynchronized), (\"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83\", TimeNotSynchronized), (\"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2\", TimeNotSynchronized), (\"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a\", TimeNotSynchronized), (\"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da\", TimeNotSynchronized), (\"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8\", TimeNotSynchronized), (\"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65\", TimeNotSynchronized), (\"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf\", TimeNotSynchronized)]","latency_us":511215,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:57.025675955Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"299.606921ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"dc71e953-3718-4311-b0c4-d25a057af8e2\", \"content-length\": \"124\", \"date\": \"Mon, 05 Aug 2024 16:43:57 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"dc71e953-3718-4311-b0c4-d25a057af8e2\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 83a194e2-3464-4a84-ae47-e33a0983bc56, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: fe17df80-010a-4b68-9ebc-6a94437be875, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5b6cee62-004c-4843-9e16-c25d2b0c743b, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6300d678-c957-4259-9a3e-c167ee4d0fb0, kind: Service { id: 5b6cee62-004c-4843-9e16-c25d2b0c743b }, name: Name(\"ntp-5b6cee62-004c-4843-9e16-c25d2b0c743b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 71])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 3c470f26-8422-4139-9f7f-81a0d84becba, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: acae5a41-5da9-420f-9bc8-154e4abdaf87, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } } } }, OmicronZoneConfig { id: eed4ad2f-7765-43e1-8943-ea458ed22e57, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } } } }, OmicronZoneConfig { id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 902ea001-1913-4130-9979-87a8d3db1f53, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8995285b-5cb0-4d3a-b021-8aec05314a65, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: c8555f5b-90c8-4b5b-b73e-5b0ba7ce0695, kind: Service { id: 8995285b-5cb0-4d3a-b021-8aec05314a65 }, name: Name(\"external-dns-8995285b-5cb0-4d3a-b021-8aec05314a65\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 233, 199])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 0a07869d-c9ec-4ade-b114-97ceff54b60e, kind: Service { id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8 }, name: Name(\"external-dns-8a344ff3-4023-4f5e-a29a-3db2dc6980b8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 206, 204])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 866d6328-4ca1-435e-8a98-cd5268df29e8, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 6eedd1d7-7b63-4139-96c3-ad58e9233e7c, kind: Service { id: 866d6328-4ca1-435e-8a98-cd5268df29e8 }, name: Name(\"nexus-866d6328-4ca1-435e-8a98-cd5268df29e8\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 250])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4952ec5d-23ba-48ea-9509-011fcb02adaf, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 38978e97-2109-4173-8624-98de2b729958, kind: Service { id: 4952ec5d-23ba-48ea-9509-011fcb02adaf }, name: Name(\"nexus-4952ec5d-23ba-48ea-9509-011fcb02adaf\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 188])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1c82ddd6-6f5a-4a92-a9b3-fde01c046c42, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 51730dfc-654f-49d5-a0b7-5f1b747fb3be, kind: Service { id: 1c82ddd6-6f5a-4a92-a9b3-fde01c046c42 }, name: Name(\"nexus-1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 28])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 893afdd2-39c6-4f94-b381-facb1a40bbb1, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 061f99f8-6626-4a81-8262-abd77ceeb36a, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 15711896-d3e4-44e3-b330-624f0f5e064d, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: c02f008f-19c7-48c5-93c6-3f776c51e015, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 738351b7-dacb-486a-ab38-cf8aefaef9cb, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 29db7ccb-90e1-414b-aeee-a9393f639631, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } } } }, OmicronZoneConfig { id: d857801a-a95b-4937-a2f0-31ad7c239ac7, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External } } } }, OmicronZoneConfig { id: 58b140e5-f61e-4410-8851-af929dfd8069, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External } } } }, OmicronZoneConfig { id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External } } } }, OmicronZoneConfig { id: c1741444-7ab7-4888-aa3c-e7b6da3527da, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:57.32802524Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83a194e2-3464-4a84-ae47-e33a0983bc56, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:57.328323407Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:57.328331582Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 83a194e2-3464-4a84-ae47-e33a0983bc56, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:57.352050314Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:57.352095812Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe17df80-010a-4b68-9ebc-6a94437be875, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:57.372322195Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:57.3723697Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: fe17df80-010a-4b68-9ebc-6a94437be875, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c470f26-8422-4139-9f7f-81a0d84becba, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:57.390976485Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:57.39108527Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 3c470f26-8422-4139-9f7f-81a0d84becba, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: acae5a41-5da9-420f-9bc8-154e4abdaf87, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:57.411183048Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:57.411221749Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: acae5a41-5da9-420f-9bc8-154e4abdaf87, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eed4ad2f-7765-43e1-8943-ea458ed22e57, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:57.432828128Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:57.432873249Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: eed4ad2f-7765-43e1-8943-ea458ed22e57, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:57.454660283Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:57.454704589Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 902ea001-1913-4130-9979-87a8d3db1f53, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:57.47430204Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:57.474344004Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 902ea001-1913-4130-9979-87a8d3db1f53, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8995285b-5cb0-4d3a-b021-8aec05314a65, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:57.492809384Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:57.492855969Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8995285b-5cb0-4d3a-b021-8aec05314a65, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:57.512783049Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:57.512824159Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 061f99f8-6626-4a81-8262-abd77ceeb36a, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:57.532025965Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:57.532084363Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 061f99f8-6626-4a81-8262-abd77ceeb36a, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 29db7ccb-90e1-414b-aeee-a9393f639631, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:57.55152712Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:57.55157124Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 29db7ccb-90e1-414b-aeee-a9393f639631, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:57.570317334Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:57.570355229Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d857801a-a95b-4937-a2f0-31ad7c239ac7, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:57.589529335Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:57.589639561Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: d857801a-a95b-4937-a2f0-31ad7c239ac7, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:57.608346046Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:57.608385863Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:57.628385054Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:57.628423127Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 58b140e5-f61e-4410-8851-af929dfd8069, dataset_name: DatasetName { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:57.646771924Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:57.646808796Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 58b140e5-f61e-4410-8851-af929dfd8069, dataset_name: DatasetName { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:57.670255069Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:57.670303135Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, dataset_name: DatasetName { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:57.692418548Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:57.69246339Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, dataset_name: DatasetName { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1741444-7ab7-4888-aa3c-e7b6da3527da, dataset_name: DatasetName { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:57.713148931Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:57.713195038Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: c1741444-7ab7-4888-aa3c-e7b6da3527da, dataset_name: DatasetName { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:57.734112585Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Reading ledger from /pool/int/4576fedf-479b-43b3-8fad-bb8b7ec274f8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:57.734158748Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/d09b3f64-0a36-4052-8233-a94e62e662e2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:57.734372174Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:57.734499845Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:57.830570819Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:57.830619607Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:57.830626301Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:57.830631935Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:57.830637401Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:57.830642567Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:57.830647685Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:57.830654042Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:57.830661092Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:57.830666539Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:57.830672024Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:57.830687989Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:57.830694777Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:57.830699699Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:57.830706308Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:57.83071153Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:57.830716808Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:57.830722227Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:57.830725921Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:57.830796076Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"463a5426-0c6b-4a6f-b23d-57b8f1452ea2","remote_addr":"[fd00:1122:3344:101::1]:59148","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451\", TimeNotSynchronized), (\"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300\", TimeNotSynchronized), (\"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65\", TimeNotSynchronized), (\"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf\", TimeNotSynchronized), (\"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8\", TimeNotSynchronized), (\"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d\", TimeNotSynchronized), (\"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a\", TimeNotSynchronized), (\"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83\", TimeNotSynchronized), (\"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069\", TimeNotSynchronized), (\"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631\", TimeNotSynchronized), (\"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2\", TimeNotSynchronized), (\"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d\", TimeNotSynchronized), (\"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da\", TimeNotSynchronized), (\"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8\", TimeNotSynchronized), (\"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1\", TimeNotSynchronized), (\"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb\", TimeNotSynchronized), (\"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015\", TimeNotSynchronized), (\"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7\", TimeNotSynchronized), (\"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\", TimeNotSynchronized)]","latency_us":501318,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:57.830948179Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"475.581437ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"463a5426-0c6b-4a6f-b23d-57b8f1452ea2\", \"content-length\": \"124\", \"date\": \"Mon, 05 Aug 2024 16:43:57 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"463a5426-0c6b-4a6f-b23d-57b8f1452ea2\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 83a194e2-3464-4a84-ae47-e33a0983bc56, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: fe17df80-010a-4b68-9ebc-6a94437be875, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5b6cee62-004c-4843-9e16-c25d2b0c743b, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6300d678-c957-4259-9a3e-c167ee4d0fb0, kind: Service { id: 5b6cee62-004c-4843-9e16-c25d2b0c743b }, name: Name(\"ntp-5b6cee62-004c-4843-9e16-c25d2b0c743b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 71])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 3c470f26-8422-4139-9f7f-81a0d84becba, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: acae5a41-5da9-420f-9bc8-154e4abdaf87, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } } } }, OmicronZoneConfig { id: eed4ad2f-7765-43e1-8943-ea458ed22e57, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } } } }, OmicronZoneConfig { id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 902ea001-1913-4130-9979-87a8d3db1f53, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8995285b-5cb0-4d3a-b021-8aec05314a65, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: c8555f5b-90c8-4b5b-b73e-5b0ba7ce0695, kind: Service { id: 8995285b-5cb0-4d3a-b021-8aec05314a65 }, name: Name(\"external-dns-8995285b-5cb0-4d3a-b021-8aec05314a65\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 233, 199])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 0a07869d-c9ec-4ade-b114-97ceff54b60e, kind: Service { id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8 }, name: Name(\"external-dns-8a344ff3-4023-4f5e-a29a-3db2dc6980b8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 206, 204])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 866d6328-4ca1-435e-8a98-cd5268df29e8, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 6eedd1d7-7b63-4139-96c3-ad58e9233e7c, kind: Service { id: 866d6328-4ca1-435e-8a98-cd5268df29e8 }, name: Name(\"nexus-866d6328-4ca1-435e-8a98-cd5268df29e8\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 250])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4952ec5d-23ba-48ea-9509-011fcb02adaf, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 38978e97-2109-4173-8624-98de2b729958, kind: Service { id: 4952ec5d-23ba-48ea-9509-011fcb02adaf }, name: Name(\"nexus-4952ec5d-23ba-48ea-9509-011fcb02adaf\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 188])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1c82ddd6-6f5a-4a92-a9b3-fde01c046c42, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 51730dfc-654f-49d5-a0b7-5f1b747fb3be, kind: Service { id: 1c82ddd6-6f5a-4a92-a9b3-fde01c046c42 }, name: Name(\"nexus-1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 28])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 893afdd2-39c6-4f94-b381-facb1a40bbb1, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 061f99f8-6626-4a81-8262-abd77ceeb36a, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 15711896-d3e4-44e3-b330-624f0f5e064d, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: c02f008f-19c7-48c5-93c6-3f776c51e015, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 738351b7-dacb-486a-ab38-cf8aefaef9cb, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 29db7ccb-90e1-414b-aeee-a9393f639631, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } } } }, OmicronZoneConfig { id: d857801a-a95b-4937-a2f0-31ad7c239ac7, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External } } } }, OmicronZoneConfig { id: 58b140e5-f61e-4410-8851-af929dfd8069, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External } } } }, OmicronZoneConfig { id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External } } } }, OmicronZoneConfig { id: c1741444-7ab7-4888-aa3c-e7b6da3527da, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:58.309795117Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83a194e2-3464-4a84-ae47-e33a0983bc56, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:58.310098498Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:58.310107104Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 83a194e2-3464-4a84-ae47-e33a0983bc56, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:58.332777726Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:58.332824883Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe17df80-010a-4b68-9ebc-6a94437be875, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:58.352734113Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:58.352778776Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: fe17df80-010a-4b68-9ebc-6a94437be875, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c470f26-8422-4139-9f7f-81a0d84becba, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:58.373021824Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:58.373062213Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 3c470f26-8422-4139-9f7f-81a0d84becba, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: acae5a41-5da9-420f-9bc8-154e4abdaf87, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:58.391184751Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:58.391229132Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: acae5a41-5da9-420f-9bc8-154e4abdaf87, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eed4ad2f-7765-43e1-8943-ea458ed22e57, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:58.410757275Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:58.410858348Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: eed4ad2f-7765-43e1-8943-ea458ed22e57, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:58.43014738Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:58.430184759Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 902ea001-1913-4130-9979-87a8d3db1f53, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:58.448808006Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:58.448848104Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 902ea001-1913-4130-9979-87a8d3db1f53, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8995285b-5cb0-4d3a-b021-8aec05314a65, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:58.467888057Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:58.467996424Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8995285b-5cb0-4d3a-b021-8aec05314a65, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:58.487564667Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:58.487604896Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 061f99f8-6626-4a81-8262-abd77ceeb36a, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:58.506701766Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:58.506739876Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 061f99f8-6626-4a81-8262-abd77ceeb36a, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 29db7ccb-90e1-414b-aeee-a9393f639631, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:58.526011104Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:58.526056011Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 29db7ccb-90e1-414b-aeee-a9393f639631, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:58.54496713Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:58.545006515Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d857801a-a95b-4937-a2f0-31ad7c239ac7, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:58.56373797Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:58.563773202Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: d857801a-a95b-4937-a2f0-31ad7c239ac7, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:58.582924455Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:58.582963408Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:58.602581499Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:58.602622666Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 58b140e5-f61e-4410-8851-af929dfd8069, dataset_name: DatasetName { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:58.622814677Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:58.622858787Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 58b140e5-f61e-4410-8851-af929dfd8069, dataset_name: DatasetName { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:58.643754849Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:58.643787342Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, dataset_name: DatasetName { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:58.664378555Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:58.664414481Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, dataset_name: DatasetName { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1741444-7ab7-4888-aa3c-e7b6da3527da, dataset_name: DatasetName { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:58.684471233Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:58.684505322Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: c1741444-7ab7-4888-aa3c-e7b6da3527da, dataset_name: DatasetName { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:58.704153705Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Reading ledger from /pool/int/4576fedf-479b-43b3-8fad-bb8b7ec274f8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:58.704197664Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/d09b3f64-0a36-4052-8233-a94e62e662e2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:58.704397037Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:58.704515876Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:58.800365852Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:58.800417631Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:58.800426584Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:58.8004328Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:58.800439363Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:58.800447566Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:58.800454372Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:58.800460776Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:58.800468989Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:58.80047511Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:58.800481635Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:58.800487907Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:58.800494245Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:58.800500517Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:58.800516754Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:58.800523402Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:58.800531483Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:58.800537727Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:58.800545733Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:58.800676844Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"05ed379d-b82e-4b40-9078-20de00baf85a","remote_addr":"[fd00:1122:3344:101::1]:59148","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83\", TimeNotSynchronized), (\"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015\", TimeNotSynchronized), (\"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d\", TimeNotSynchronized), (\"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069\", TimeNotSynchronized), (\"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a\", TimeNotSynchronized), (\"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300\", TimeNotSynchronized), (\"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451\", TimeNotSynchronized), (\"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da\", TimeNotSynchronized), (\"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\", TimeNotSynchronized), (\"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631\", TimeNotSynchronized), (\"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65\", TimeNotSynchronized), (\"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8\", TimeNotSynchronized), (\"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d\", TimeNotSynchronized), (\"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb\", TimeNotSynchronized), (\"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1\", TimeNotSynchronized), (\"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7\", TimeNotSynchronized), (\"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2\", TimeNotSynchronized), (\"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8\", TimeNotSynchronized), (\"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf\", TimeNotSynchronized)]","latency_us":489458,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:58.80091493Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"458.479111ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"05ed379d-b82e-4b40-9078-20de00baf85a\", \"content-length\": \"124\", \"date\": \"Mon, 05 Aug 2024 16:43:58 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"05ed379d-b82e-4b40-9078-20de00baf85a\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 83a194e2-3464-4a84-ae47-e33a0983bc56, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: fe17df80-010a-4b68-9ebc-6a94437be875, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5b6cee62-004c-4843-9e16-c25d2b0c743b, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6300d678-c957-4259-9a3e-c167ee4d0fb0, kind: Service { id: 5b6cee62-004c-4843-9e16-c25d2b0c743b }, name: Name(\"ntp-5b6cee62-004c-4843-9e16-c25d2b0c743b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 71])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 3c470f26-8422-4139-9f7f-81a0d84becba, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: acae5a41-5da9-420f-9bc8-154e4abdaf87, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } } } }, OmicronZoneConfig { id: eed4ad2f-7765-43e1-8943-ea458ed22e57, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } } } }, OmicronZoneConfig { id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 902ea001-1913-4130-9979-87a8d3db1f53, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8995285b-5cb0-4d3a-b021-8aec05314a65, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: c8555f5b-90c8-4b5b-b73e-5b0ba7ce0695, kind: Service { id: 8995285b-5cb0-4d3a-b021-8aec05314a65 }, name: Name(\"external-dns-8995285b-5cb0-4d3a-b021-8aec05314a65\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 233, 199])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 0a07869d-c9ec-4ade-b114-97ceff54b60e, kind: Service { id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8 }, name: Name(\"external-dns-8a344ff3-4023-4f5e-a29a-3db2dc6980b8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 206, 204])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 866d6328-4ca1-435e-8a98-cd5268df29e8, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 6eedd1d7-7b63-4139-96c3-ad58e9233e7c, kind: Service { id: 866d6328-4ca1-435e-8a98-cd5268df29e8 }, name: Name(\"nexus-866d6328-4ca1-435e-8a98-cd5268df29e8\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 250])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4952ec5d-23ba-48ea-9509-011fcb02adaf, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 38978e97-2109-4173-8624-98de2b729958, kind: Service { id: 4952ec5d-23ba-48ea-9509-011fcb02adaf }, name: Name(\"nexus-4952ec5d-23ba-48ea-9509-011fcb02adaf\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 188])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1c82ddd6-6f5a-4a92-a9b3-fde01c046c42, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 51730dfc-654f-49d5-a0b7-5f1b747fb3be, kind: Service { id: 1c82ddd6-6f5a-4a92-a9b3-fde01c046c42 }, name: Name(\"nexus-1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 28])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 893afdd2-39c6-4f94-b381-facb1a40bbb1, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 061f99f8-6626-4a81-8262-abd77ceeb36a, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 15711896-d3e4-44e3-b330-624f0f5e064d, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: c02f008f-19c7-48c5-93c6-3f776c51e015, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 738351b7-dacb-486a-ab38-cf8aefaef9cb, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 29db7ccb-90e1-414b-aeee-a9393f639631, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } } } }, OmicronZoneConfig { id: d857801a-a95b-4937-a2f0-31ad7c239ac7, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External } } } }, OmicronZoneConfig { id: 58b140e5-f61e-4410-8851-af929dfd8069, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External } } } }, OmicronZoneConfig { id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External } } } }, OmicronZoneConfig { id: c1741444-7ab7-4888-aa3c-e7b6da3527da, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:59.262448961Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83a194e2-3464-4a84-ae47-e33a0983bc56, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:59.262791407Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:59.262799713Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 83a194e2-3464-4a84-ae47-e33a0983bc56, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:59.285214881Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:59.285262384Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe17df80-010a-4b68-9ebc-6a94437be875, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:59.30518578Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:59.305291003Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: fe17df80-010a-4b68-9ebc-6a94437be875, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c470f26-8422-4139-9f7f-81a0d84becba, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:59.324606971Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:59.324657522Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 3c470f26-8422-4139-9f7f-81a0d84becba, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: acae5a41-5da9-420f-9bc8-154e4abdaf87, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:59.343800204Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:59.343836851Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: acae5a41-5da9-420f-9bc8-154e4abdaf87, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eed4ad2f-7765-43e1-8943-ea458ed22e57, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:59.363188228Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:59.363223244Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: eed4ad2f-7765-43e1-8943-ea458ed22e57, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:59.382560707Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:59.382599707Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 902ea001-1913-4130-9979-87a8d3db1f53, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:59.401358416Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:59.401397867Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 902ea001-1913-4130-9979-87a8d3db1f53, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8995285b-5cb0-4d3a-b021-8aec05314a65, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:59.420453762Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:59.420496598Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8995285b-5cb0-4d3a-b021-8aec05314a65, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:59.441152199Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:59.441198324Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 061f99f8-6626-4a81-8262-abd77ceeb36a, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:59.460876162Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:59.460911216Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 061f99f8-6626-4a81-8262-abd77ceeb36a, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 29db7ccb-90e1-414b-aeee-a9393f639631, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:59.479672747Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:59.479780032Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 29db7ccb-90e1-414b-aeee-a9393f639631, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:59.49903578Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:59.499076018Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d857801a-a95b-4937-a2f0-31ad7c239ac7, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:59.518389038Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:59.518427711Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: d857801a-a95b-4937-a2f0-31ad7c239ac7, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:59.537083686Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:59.537117736Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:59.55584076Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:59.555874689Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 58b140e5-f61e-4410-8851-af929dfd8069, dataset_name: DatasetName { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:59.575088598Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:59.575124796Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 58b140e5-f61e-4410-8851-af929dfd8069, dataset_name: DatasetName { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:59.593764321Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:59.593808994Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, dataset_name: DatasetName { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:59.612156873Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:59.612189602Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, dataset_name: DatasetName { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1741444-7ab7-4888-aa3c-e7b6da3527da, dataset_name: DatasetName { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:59.63090341Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:59.630940902Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: c1741444-7ab7-4888-aa3c-e7b6da3527da, dataset_name: DatasetName { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:59.65129988Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Reading ledger from /pool/int/4576fedf-479b-43b3-8fad-bb8b7ec274f8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:59.651348368Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/d09b3f64-0a36-4052-8233-a94e62e662e2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:59.651547187Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:59.651662257Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:59.746888336Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:59.746938728Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:59.74694844Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:59.746976576Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:59.746984938Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:59.746991276Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:59.746998776Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:59.747005057Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:59.747011169Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:59.747018707Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:59.747024857Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:59.747031195Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:59.747035227Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:59.747041255Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:59.747046036Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:59.747051961Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:59.747058177Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:59.747065761Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:59.747077106Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:43:59.747145282Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"a32214d9-5b62-4c32-9d26-11743c0cf571","remote_addr":"[fd00:1122:3344:101::1]:59148","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1\", TimeNotSynchronized), (\"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a\", TimeNotSynchronized), (\"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015\", TimeNotSynchronized), (\"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf\", TimeNotSynchronized), (\"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631\", TimeNotSynchronized), (\"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069\", TimeNotSynchronized), (\"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451\", TimeNotSynchronized), (\"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8\", TimeNotSynchronized), (\"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8\", TimeNotSynchronized), (\"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb\", TimeNotSynchronized), (\"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83\", TimeNotSynchronized), (\"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300\", TimeNotSynchronized), (\"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d\", TimeNotSynchronized), (\"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da\", TimeNotSynchronized), (\"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65\", TimeNotSynchronized), (\"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7\", TimeNotSynchronized), (\"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2\", TimeNotSynchronized), (\"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d\", TimeNotSynchronized), (\"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\", TimeNotSynchronized)]","latency_us":483270,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:43:59.747340875Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"584.521444ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"a32214d9-5b62-4c32-9d26-11743c0cf571\", \"content-length\": \"124\", \"date\": \"Mon, 05 Aug 2024 16:43:59 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"a32214d9-5b62-4c32-9d26-11743c0cf571\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 83a194e2-3464-4a84-ae47-e33a0983bc56, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: fe17df80-010a-4b68-9ebc-6a94437be875, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5b6cee62-004c-4843-9e16-c25d2b0c743b, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6300d678-c957-4259-9a3e-c167ee4d0fb0, kind: Service { id: 5b6cee62-004c-4843-9e16-c25d2b0c743b }, name: Name(\"ntp-5b6cee62-004c-4843-9e16-c25d2b0c743b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 71])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 3c470f26-8422-4139-9f7f-81a0d84becba, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: acae5a41-5da9-420f-9bc8-154e4abdaf87, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } } } }, OmicronZoneConfig { id: eed4ad2f-7765-43e1-8943-ea458ed22e57, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } } } }, OmicronZoneConfig { id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 902ea001-1913-4130-9979-87a8d3db1f53, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8995285b-5cb0-4d3a-b021-8aec05314a65, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: c8555f5b-90c8-4b5b-b73e-5b0ba7ce0695, kind: Service { id: 8995285b-5cb0-4d3a-b021-8aec05314a65 }, name: Name(\"external-dns-8995285b-5cb0-4d3a-b021-8aec05314a65\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 233, 199])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 0a07869d-c9ec-4ade-b114-97ceff54b60e, kind: Service { id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8 }, name: Name(\"external-dns-8a344ff3-4023-4f5e-a29a-3db2dc6980b8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 206, 204])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 866d6328-4ca1-435e-8a98-cd5268df29e8, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 6eedd1d7-7b63-4139-96c3-ad58e9233e7c, kind: Service { id: 866d6328-4ca1-435e-8a98-cd5268df29e8 }, name: Name(\"nexus-866d6328-4ca1-435e-8a98-cd5268df29e8\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 250])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4952ec5d-23ba-48ea-9509-011fcb02adaf, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 38978e97-2109-4173-8624-98de2b729958, kind: Service { id: 4952ec5d-23ba-48ea-9509-011fcb02adaf }, name: Name(\"nexus-4952ec5d-23ba-48ea-9509-011fcb02adaf\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 188])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1c82ddd6-6f5a-4a92-a9b3-fde01c046c42, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 51730dfc-654f-49d5-a0b7-5f1b747fb3be, kind: Service { id: 1c82ddd6-6f5a-4a92-a9b3-fde01c046c42 }, name: Name(\"nexus-1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 28])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 893afdd2-39c6-4f94-b381-facb1a40bbb1, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 061f99f8-6626-4a81-8262-abd77ceeb36a, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 15711896-d3e4-44e3-b330-624f0f5e064d, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: c02f008f-19c7-48c5-93c6-3f776c51e015, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 738351b7-dacb-486a-ab38-cf8aefaef9cb, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 29db7ccb-90e1-414b-aeee-a9393f639631, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } } } }, OmicronZoneConfig { id: d857801a-a95b-4937-a2f0-31ad7c239ac7, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External } } } }, OmicronZoneConfig { id: 58b140e5-f61e-4410-8851-af929dfd8069, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External } } } }, OmicronZoneConfig { id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External } } } }, OmicronZoneConfig { id: c1741444-7ab7-4888-aa3c-e7b6da3527da, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:00.335258217Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83a194e2-3464-4a84-ae47-e33a0983bc56, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:00.335645468Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:00.335652612Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 83a194e2-3464-4a84-ae47-e33a0983bc56, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:00.358228259Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:00.358268582Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe17df80-010a-4b68-9ebc-6a94437be875, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:00.378233611Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:00.378275828Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: fe17df80-010a-4b68-9ebc-6a94437be875, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c470f26-8422-4139-9f7f-81a0d84becba, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:00.397940203Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:00.397977273Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 3c470f26-8422-4139-9f7f-81a0d84becba, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: acae5a41-5da9-420f-9bc8-154e4abdaf87, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:00.418445742Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:00.418500024Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: acae5a41-5da9-420f-9bc8-154e4abdaf87, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eed4ad2f-7765-43e1-8943-ea458ed22e57, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:00.437368004Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:00.43740723Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: eed4ad2f-7765-43e1-8943-ea458ed22e57, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:00.456204186Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:00.456237046Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 902ea001-1913-4130-9979-87a8d3db1f53, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:00.475315085Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:00.475349202Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 902ea001-1913-4130-9979-87a8d3db1f53, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8995285b-5cb0-4d3a-b021-8aec05314a65, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:00.493360542Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:00.493392812Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8995285b-5cb0-4d3a-b021-8aec05314a65, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:00.513107229Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:00.51314758Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 061f99f8-6626-4a81-8262-abd77ceeb36a, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:00.531889853Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:00.531922553Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 061f99f8-6626-4a81-8262-abd77ceeb36a, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 29db7ccb-90e1-414b-aeee-a9393f639631, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:00.550947802Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:00.550981009Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 29db7ccb-90e1-414b-aeee-a9393f639631, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:00.570492368Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:00.570525537Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d857801a-a95b-4937-a2f0-31ad7c239ac7, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:00.590821856Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:00.59085526Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: d857801a-a95b-4937-a2f0-31ad7c239ac7, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:00.610513182Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:00.610550271Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:00.630273401Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:00.630312186Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 58b140e5-f61e-4410-8851-af929dfd8069, dataset_name: DatasetName { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:00.651116074Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:00.651152291Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 58b140e5-f61e-4410-8851-af929dfd8069, dataset_name: DatasetName { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:00.671651815Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:00.671682229Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, dataset_name: DatasetName { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:00.690588533Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:00.690630318Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, dataset_name: DatasetName { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1741444-7ab7-4888-aa3c-e7b6da3527da, dataset_name: DatasetName { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:00.709560403Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:00.709592896Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: c1741444-7ab7-4888-aa3c-e7b6da3527da, dataset_name: DatasetName { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:00.729230368Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Reading ledger from /pool/int/4576fedf-479b-43b3-8fad-bb8b7ec274f8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:00.729267841Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/d09b3f64-0a36-4052-8233-a94e62e662e2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:00.729453056Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:00.72956463Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:00.829305237Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:00.829416044Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:00.829421566Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:00.829426779Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:00.829430529Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:00.829437841Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:00.829444629Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:00.829465864Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:00.829472398Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:00.829477751Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:00.829482805Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:00.829487801Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:00.829491336Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:00.829494898Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:00.829499876Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:00.829504742Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:00.829510105Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:00.829514717Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:00.829520934Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:00.829573125Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"b43927c7-7631-40f0-9ea6-b15ba4077f44","remote_addr":"[fd00:1122:3344:101::1]:59148","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631\", TimeNotSynchronized), (\"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\", TimeNotSynchronized), (\"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015\", TimeNotSynchronized), (\"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1\", TimeNotSynchronized), (\"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65\", TimeNotSynchronized), (\"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8\", TimeNotSynchronized), (\"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7\", TimeNotSynchronized), (\"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069\", TimeNotSynchronized), (\"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451\", TimeNotSynchronized), (\"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d\", TimeNotSynchronized), (\"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83\", TimeNotSynchronized), (\"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300\", TimeNotSynchronized), (\"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da\", TimeNotSynchronized), (\"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d\", TimeNotSynchronized), (\"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf\", TimeNotSynchronized), (\"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a\", TimeNotSynchronized), (\"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8\", TimeNotSynchronized), (\"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb\", TimeNotSynchronized), (\"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2\", TimeNotSynchronized)]","latency_us":492811,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:00.829750381Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"753.966264ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b43927c7-7631-40f0-9ea6-b15ba4077f44\", \"content-length\": \"124\", \"date\": \"Mon, 05 Aug 2024 16:44:00 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b43927c7-7631-40f0-9ea6-b15ba4077f44\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 83a194e2-3464-4a84-ae47-e33a0983bc56, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: fe17df80-010a-4b68-9ebc-6a94437be875, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5b6cee62-004c-4843-9e16-c25d2b0c743b, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6300d678-c957-4259-9a3e-c167ee4d0fb0, kind: Service { id: 5b6cee62-004c-4843-9e16-c25d2b0c743b }, name: Name(\"ntp-5b6cee62-004c-4843-9e16-c25d2b0c743b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 71])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 3c470f26-8422-4139-9f7f-81a0d84becba, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: acae5a41-5da9-420f-9bc8-154e4abdaf87, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } } } }, OmicronZoneConfig { id: eed4ad2f-7765-43e1-8943-ea458ed22e57, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } } } }, OmicronZoneConfig { id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 902ea001-1913-4130-9979-87a8d3db1f53, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8995285b-5cb0-4d3a-b021-8aec05314a65, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: c8555f5b-90c8-4b5b-b73e-5b0ba7ce0695, kind: Service { id: 8995285b-5cb0-4d3a-b021-8aec05314a65 }, name: Name(\"external-dns-8995285b-5cb0-4d3a-b021-8aec05314a65\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 233, 199])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 0a07869d-c9ec-4ade-b114-97ceff54b60e, kind: Service { id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8 }, name: Name(\"external-dns-8a344ff3-4023-4f5e-a29a-3db2dc6980b8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 206, 204])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 866d6328-4ca1-435e-8a98-cd5268df29e8, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 6eedd1d7-7b63-4139-96c3-ad58e9233e7c, kind: Service { id: 866d6328-4ca1-435e-8a98-cd5268df29e8 }, name: Name(\"nexus-866d6328-4ca1-435e-8a98-cd5268df29e8\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 250])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4952ec5d-23ba-48ea-9509-011fcb02adaf, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 38978e97-2109-4173-8624-98de2b729958, kind: Service { id: 4952ec5d-23ba-48ea-9509-011fcb02adaf }, name: Name(\"nexus-4952ec5d-23ba-48ea-9509-011fcb02adaf\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 188])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1c82ddd6-6f5a-4a92-a9b3-fde01c046c42, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 51730dfc-654f-49d5-a0b7-5f1b747fb3be, kind: Service { id: 1c82ddd6-6f5a-4a92-a9b3-fde01c046c42 }, name: Name(\"nexus-1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 28])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 893afdd2-39c6-4f94-b381-facb1a40bbb1, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 061f99f8-6626-4a81-8262-abd77ceeb36a, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 15711896-d3e4-44e3-b330-624f0f5e064d, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: c02f008f-19c7-48c5-93c6-3f776c51e015, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 738351b7-dacb-486a-ab38-cf8aefaef9cb, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 29db7ccb-90e1-414b-aeee-a9393f639631, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } } } }, OmicronZoneConfig { id: d857801a-a95b-4937-a2f0-31ad7c239ac7, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External } } } }, OmicronZoneConfig { id: 58b140e5-f61e-4410-8851-af929dfd8069, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External } } } }, OmicronZoneConfig { id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External } } } }, OmicronZoneConfig { id: c1741444-7ab7-4888-aa3c-e7b6da3527da, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:01.587473295Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83a194e2-3464-4a84-ae47-e33a0983bc56, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:01.587764421Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:01.587772465Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 83a194e2-3464-4a84-ae47-e33a0983bc56, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:01.608634527Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:01.60866478Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe17df80-010a-4b68-9ebc-6a94437be875, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:01.626594696Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:01.626619963Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: fe17df80-010a-4b68-9ebc-6a94437be875, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c470f26-8422-4139-9f7f-81a0d84becba, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:01.644783675Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:01.644811023Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 3c470f26-8422-4139-9f7f-81a0d84becba, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: acae5a41-5da9-420f-9bc8-154e4abdaf87, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:01.662532265Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:01.662560457Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: acae5a41-5da9-420f-9bc8-154e4abdaf87, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eed4ad2f-7765-43e1-8943-ea458ed22e57, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:01.683024117Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:01.683052627Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: eed4ad2f-7765-43e1-8943-ea458ed22e57, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:01.701956157Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:01.701993235Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 902ea001-1913-4130-9979-87a8d3db1f53, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:01.720591753Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:01.720625017Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 902ea001-1913-4130-9979-87a8d3db1f53, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8995285b-5cb0-4d3a-b021-8aec05314a65, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:01.740223895Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:01.740255293Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8995285b-5cb0-4d3a-b021-8aec05314a65, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:01.759104121Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:01.759135921Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 061f99f8-6626-4a81-8262-abd77ceeb36a, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:01.777910141Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:01.77801405Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 061f99f8-6626-4a81-8262-abd77ceeb36a, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 29db7ccb-90e1-414b-aeee-a9393f639631, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:01.795879Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:01.795915704Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 29db7ccb-90e1-414b-aeee-a9393f639631, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:01.813796525Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:01.813823919Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d857801a-a95b-4937-a2f0-31ad7c239ac7, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:01.831691731Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:01.831725622Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: d857801a-a95b-4937-a2f0-31ad7c239ac7, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:01.850075982Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:01.850105607Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:01.867878741Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:01.867907064Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 58b140e5-f61e-4410-8851-af929dfd8069, dataset_name: DatasetName { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:01.886148971Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:01.886183426Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 58b140e5-f61e-4410-8851-af929dfd8069, dataset_name: DatasetName { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:01.90456035Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:01.904588991Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, dataset_name: DatasetName { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:01.924105122Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:01.924137475Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, dataset_name: DatasetName { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1741444-7ab7-4888-aa3c-e7b6da3527da, dataset_name: DatasetName { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:01.942139931Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:01.942175538Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: c1741444-7ab7-4888-aa3c-e7b6da3527da, dataset_name: DatasetName { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:01.961546562Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Reading ledger from /pool/int/4576fedf-479b-43b3-8fad-bb8b7ec274f8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:01.961591938Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/d09b3f64-0a36-4052-8233-a94e62e662e2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:01.961779881Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:01.961893442Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:02.056751845Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:02.056811762Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:02.056821156Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:02.056829715Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:02.056837749Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:02.056844247Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:02.056852141Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:02.056860025Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:02.056864778Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:02.056873469Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:02.056885947Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:02.056923645Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:02.056932289Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:02.056938673Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:02.056945095Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:02.056949539Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:02.05695582Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:02.056963443Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:02.056971196Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:02.057027362Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"d143f344-9a5c-471e-9277-71af0db223c5","remote_addr":"[fd00:1122:3344:101::1]:59148","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\", TimeNotSynchronized), (\"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb\", TimeNotSynchronized), (\"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1\", TimeNotSynchronized), (\"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf\", TimeNotSynchronized), (\"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a\", TimeNotSynchronized), (\"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069\", TimeNotSynchronized), (\"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65\", TimeNotSynchronized), (\"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300\", TimeNotSynchronized), (\"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d\", TimeNotSynchronized), (\"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015\", TimeNotSynchronized), (\"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83\", TimeNotSynchronized), (\"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8\", TimeNotSynchronized), (\"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7\", TimeNotSynchronized), (\"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8\", TimeNotSynchronized), (\"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2\", TimeNotSynchronized), (\"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451\", TimeNotSynchronized), (\"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631\", TimeNotSynchronized), (\"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d\", TimeNotSynchronized), (\"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da\", TimeNotSynchronized)]","latency_us":468179,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:02.057175255Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"708.394841ms","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"d143f344-9a5c-471e-9277-71af0db223c5\", \"content-length\": \"124\", \"date\": \"Mon, 05 Aug 2024 16:44:02 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"d143f344-9a5c-471e-9277-71af0db223c5\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 83a194e2-3464-4a84-ae47-e33a0983bc56, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: fe17df80-010a-4b68-9ebc-6a94437be875, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5b6cee62-004c-4843-9e16-c25d2b0c743b, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6300d678-c957-4259-9a3e-c167ee4d0fb0, kind: Service { id: 5b6cee62-004c-4843-9e16-c25d2b0c743b }, name: Name(\"ntp-5b6cee62-004c-4843-9e16-c25d2b0c743b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 71])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 3c470f26-8422-4139-9f7f-81a0d84becba, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: acae5a41-5da9-420f-9bc8-154e4abdaf87, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } } } }, OmicronZoneConfig { id: eed4ad2f-7765-43e1-8943-ea458ed22e57, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } } } }, OmicronZoneConfig { id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 902ea001-1913-4130-9979-87a8d3db1f53, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8995285b-5cb0-4d3a-b021-8aec05314a65, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: c8555f5b-90c8-4b5b-b73e-5b0ba7ce0695, kind: Service { id: 8995285b-5cb0-4d3a-b021-8aec05314a65 }, name: Name(\"external-dns-8995285b-5cb0-4d3a-b021-8aec05314a65\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 233, 199])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 0a07869d-c9ec-4ade-b114-97ceff54b60e, kind: Service { id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8 }, name: Name(\"external-dns-8a344ff3-4023-4f5e-a29a-3db2dc6980b8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 206, 204])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 866d6328-4ca1-435e-8a98-cd5268df29e8, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 6eedd1d7-7b63-4139-96c3-ad58e9233e7c, kind: Service { id: 866d6328-4ca1-435e-8a98-cd5268df29e8 }, name: Name(\"nexus-866d6328-4ca1-435e-8a98-cd5268df29e8\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 250])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4952ec5d-23ba-48ea-9509-011fcb02adaf, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 38978e97-2109-4173-8624-98de2b729958, kind: Service { id: 4952ec5d-23ba-48ea-9509-011fcb02adaf }, name: Name(\"nexus-4952ec5d-23ba-48ea-9509-011fcb02adaf\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 188])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1c82ddd6-6f5a-4a92-a9b3-fde01c046c42, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 51730dfc-654f-49d5-a0b7-5f1b747fb3be, kind: Service { id: 1c82ddd6-6f5a-4a92-a9b3-fde01c046c42 }, name: Name(\"nexus-1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 28])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 893afdd2-39c6-4f94-b381-facb1a40bbb1, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 061f99f8-6626-4a81-8262-abd77ceeb36a, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 15711896-d3e4-44e3-b330-624f0f5e064d, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: c02f008f-19c7-48c5-93c6-3f776c51e015, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 738351b7-dacb-486a-ab38-cf8aefaef9cb, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 29db7ccb-90e1-414b-aeee-a9393f639631, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } } } }, OmicronZoneConfig { id: d857801a-a95b-4937-a2f0-31ad7c239ac7, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External } } } }, OmicronZoneConfig { id: 58b140e5-f61e-4410-8851-af929dfd8069, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External } } } }, OmicronZoneConfig { id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External } } } }, OmicronZoneConfig { id: c1741444-7ab7-4888-aa3c-e7b6da3527da, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:02.768759349Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83a194e2-3464-4a84-ae47-e33a0983bc56, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:02.769087198Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:02.769094764Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 83a194e2-3464-4a84-ae47-e33a0983bc56, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:02.790940001Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:02.790981046Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe17df80-010a-4b68-9ebc-6a94437be875, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:02.811317712Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:02.811360032Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: fe17df80-010a-4b68-9ebc-6a94437be875, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c470f26-8422-4139-9f7f-81a0d84becba, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:02.829580694Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:02.829623089Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 3c470f26-8422-4139-9f7f-81a0d84becba, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: acae5a41-5da9-420f-9bc8-154e4abdaf87, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:02.847175293Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:02.847214041Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: acae5a41-5da9-420f-9bc8-154e4abdaf87, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eed4ad2f-7765-43e1-8943-ea458ed22e57, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:02.865069282Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:02.865105695Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: eed4ad2f-7765-43e1-8943-ea458ed22e57, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:02.884663706Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:02.884698281Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 902ea001-1913-4130-9979-87a8d3db1f53, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:02.903410388Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:02.903447214Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 902ea001-1913-4130-9979-87a8d3db1f53, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8995285b-5cb0-4d3a-b021-8aec05314a65, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:02.921388104Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:02.921426354Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8995285b-5cb0-4d3a-b021-8aec05314a65, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:02.940427978Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:02.940471506Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 061f99f8-6626-4a81-8262-abd77ceeb36a, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:02.961855898Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:02.961895302Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 061f99f8-6626-4a81-8262-abd77ceeb36a, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 29db7ccb-90e1-414b-aeee-a9393f639631, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:02.981366134Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:02.981410601Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 29db7ccb-90e1-414b-aeee-a9393f639631, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:02.999850098Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:02.999885874Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d857801a-a95b-4937-a2f0-31ad7c239ac7, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:03.018673742Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:03.018708834Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: d857801a-a95b-4937-a2f0-31ad7c239ac7, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:03.037835865Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:03.037869785Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:03.057239892Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:03.057272508Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 58b140e5-f61e-4410-8851-af929dfd8069, dataset_name: DatasetName { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:03.078517544Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:03.078564766Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 58b140e5-f61e-4410-8851-af929dfd8069, dataset_name: DatasetName { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:03.099489907Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:03.099585143Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, dataset_name: DatasetName { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:03.118806157Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:03.118835811Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, dataset_name: DatasetName { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1741444-7ab7-4888-aa3c-e7b6da3527da, dataset_name: DatasetName { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:03.137513407Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:03.137558221Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: c1741444-7ab7-4888-aa3c-e7b6da3527da, dataset_name: DatasetName { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:03.158405588Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Reading ledger from /pool/int/4576fedf-479b-43b3-8fad-bb8b7ec274f8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:03.158451677Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/d09b3f64-0a36-4052-8233-a94e62e662e2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:03.158767537Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:03.158900636Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:03.252730598Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:03.252776911Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:03.252784392Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:03.252789989Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:03.252796936Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:03.252803724Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:03.25280918Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:03.25281577Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:03.252822342Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:03.252827424Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:03.252833968Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:03.25283769Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:03.252842987Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:03.252846653Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:03.252851668Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:03.252873831Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:03.252878997Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:03.25288376Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:03.25288871Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:03.252997389Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"3e33f5b7-9f8c-47f1-b2c8-06e0dfdcd6fc","remote_addr":"[fd00:1122:3344:101::1]:59148","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83\", TimeNotSynchronized), (\"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8\", TimeNotSynchronized), (\"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2\", TimeNotSynchronized), (\"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7\", TimeNotSynchronized), (\"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d\", TimeNotSynchronized), (\"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65\", TimeNotSynchronized), (\"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1\", TimeNotSynchronized), (\"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8\", TimeNotSynchronized), (\"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631\", TimeNotSynchronized), (\"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069\", TimeNotSynchronized), (\"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451\", TimeNotSynchronized), (\"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300\", TimeNotSynchronized), (\"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da\", TimeNotSynchronized), (\"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb\", TimeNotSynchronized), (\"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\", TimeNotSynchronized), (\"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf\", TimeNotSynchronized), (\"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a\", TimeNotSynchronized), (\"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d\", TimeNotSynchronized), (\"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015\", TimeNotSynchronized)]","latency_us":482788,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:03.253148862Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"1.008121169s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3e33f5b7-9f8c-47f1-b2c8-06e0dfdcd6fc\", \"content-length\": \"124\", \"date\": \"Mon, 05 Aug 2024 16:44:03 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"3e33f5b7-9f8c-47f1-b2c8-06e0dfdcd6fc\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 83a194e2-3464-4a84-ae47-e33a0983bc56, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: fe17df80-010a-4b68-9ebc-6a94437be875, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5b6cee62-004c-4843-9e16-c25d2b0c743b, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6300d678-c957-4259-9a3e-c167ee4d0fb0, kind: Service { id: 5b6cee62-004c-4843-9e16-c25d2b0c743b }, name: Name(\"ntp-5b6cee62-004c-4843-9e16-c25d2b0c743b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 71])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 3c470f26-8422-4139-9f7f-81a0d84becba, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: acae5a41-5da9-420f-9bc8-154e4abdaf87, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } } } }, OmicronZoneConfig { id: eed4ad2f-7765-43e1-8943-ea458ed22e57, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } } } }, OmicronZoneConfig { id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 902ea001-1913-4130-9979-87a8d3db1f53, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8995285b-5cb0-4d3a-b021-8aec05314a65, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: c8555f5b-90c8-4b5b-b73e-5b0ba7ce0695, kind: Service { id: 8995285b-5cb0-4d3a-b021-8aec05314a65 }, name: Name(\"external-dns-8995285b-5cb0-4d3a-b021-8aec05314a65\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 233, 199])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 0a07869d-c9ec-4ade-b114-97ceff54b60e, kind: Service { id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8 }, name: Name(\"external-dns-8a344ff3-4023-4f5e-a29a-3db2dc6980b8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 206, 204])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 866d6328-4ca1-435e-8a98-cd5268df29e8, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 6eedd1d7-7b63-4139-96c3-ad58e9233e7c, kind: Service { id: 866d6328-4ca1-435e-8a98-cd5268df29e8 }, name: Name(\"nexus-866d6328-4ca1-435e-8a98-cd5268df29e8\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 250])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4952ec5d-23ba-48ea-9509-011fcb02adaf, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 38978e97-2109-4173-8624-98de2b729958, kind: Service { id: 4952ec5d-23ba-48ea-9509-011fcb02adaf }, name: Name(\"nexus-4952ec5d-23ba-48ea-9509-011fcb02adaf\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 188])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1c82ddd6-6f5a-4a92-a9b3-fde01c046c42, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 51730dfc-654f-49d5-a0b7-5f1b747fb3be, kind: Service { id: 1c82ddd6-6f5a-4a92-a9b3-fde01c046c42 }, name: Name(\"nexus-1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 28])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 893afdd2-39c6-4f94-b381-facb1a40bbb1, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 061f99f8-6626-4a81-8262-abd77ceeb36a, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 15711896-d3e4-44e3-b330-624f0f5e064d, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: c02f008f-19c7-48c5-93c6-3f776c51e015, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 738351b7-dacb-486a-ab38-cf8aefaef9cb, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 29db7ccb-90e1-414b-aeee-a9393f639631, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } } } }, OmicronZoneConfig { id: d857801a-a95b-4937-a2f0-31ad7c239ac7, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External } } } }, OmicronZoneConfig { id: 58b140e5-f61e-4410-8851-af929dfd8069, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External } } } }, OmicronZoneConfig { id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External } } } }, OmicronZoneConfig { id: c1741444-7ab7-4888-aa3c-e7b6da3527da, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:04.265380167Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83a194e2-3464-4a84-ae47-e33a0983bc56, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:04.266246955Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:04.26625408Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 83a194e2-3464-4a84-ae47-e33a0983bc56, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:04.289110338Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:04.289160908Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe17df80-010a-4b68-9ebc-6a94437be875, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:04.307499178Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:04.307540663Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: fe17df80-010a-4b68-9ebc-6a94437be875, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c470f26-8422-4139-9f7f-81a0d84becba, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:04.327520875Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:04.327571341Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 3c470f26-8422-4139-9f7f-81a0d84becba, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: acae5a41-5da9-420f-9bc8-154e4abdaf87, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:04.348344894Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:04.348396017Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: acae5a41-5da9-420f-9bc8-154e4abdaf87, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eed4ad2f-7765-43e1-8943-ea458ed22e57, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:04.367138417Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:04.367173903Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: eed4ad2f-7765-43e1-8943-ea458ed22e57, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:04.386823718Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:04.386872994Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 902ea001-1913-4130-9979-87a8d3db1f53, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:04.405742824Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:04.405784122Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 902ea001-1913-4130-9979-87a8d3db1f53, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8995285b-5cb0-4d3a-b021-8aec05314a65, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:04.424530997Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:04.424567289Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8995285b-5cb0-4d3a-b021-8aec05314a65, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:04.444096344Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:04.444133573Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 061f99f8-6626-4a81-8262-abd77ceeb36a, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:04.463914751Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:04.463957417Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 061f99f8-6626-4a81-8262-abd77ceeb36a, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 29db7ccb-90e1-414b-aeee-a9393f639631, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:04.483827793Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:04.483866315Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 29db7ccb-90e1-414b-aeee-a9393f639631, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:04.502263391Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:04.502369163Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d857801a-a95b-4937-a2f0-31ad7c239ac7, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:04.520548461Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:04.520582567Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: d857801a-a95b-4937-a2f0-31ad7c239ac7, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:04.539541715Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:04.539577228Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:04.560181871Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:04.560221294Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 58b140e5-f61e-4410-8851-af929dfd8069, dataset_name: DatasetName { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:04.579251704Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:04.579285623Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 58b140e5-f61e-4410-8851-af929dfd8069, dataset_name: DatasetName { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:04.597900942Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:04.597948811Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, dataset_name: DatasetName { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:04.616395631Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:04.616435063Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, dataset_name: DatasetName { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1741444-7ab7-4888-aa3c-e7b6da3527da, dataset_name: DatasetName { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:04.634874117Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:04.634909959Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: c1741444-7ab7-4888-aa3c-e7b6da3527da, dataset_name: DatasetName { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:04.653089775Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Reading ledger from /pool/int/4576fedf-479b-43b3-8fad-bb8b7ec274f8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:04.653129329Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/d09b3f64-0a36-4052-8233-a94e62e662e2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:04.653321772Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:04.653438072Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:04.752164517Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:04.752216212Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:04.752225924Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:04.752242846Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:04.752249728Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:04.752256722Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:04.752261006Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:04.752268169Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:04.752273606Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:04.752277281Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:04.752283891Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:04.752289216Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:04.752295975Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:04.752302763Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:04.752309569Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:04.752314848Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:04.752321589Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:04.752326961Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:04.752346995Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:04.752420965Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"2c5a56b7-c7e9-4e2e-8b17-47af1789af0a","remote_addr":"[fd00:1122:3344:101::1]:59148","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da\", TimeNotSynchronized), (\"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb\", TimeNotSynchronized), (\"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631\", TimeNotSynchronized), (\"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1\", TimeNotSynchronized), (\"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015\", TimeNotSynchronized), (\"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7\", TimeNotSynchronized), (\"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2\", TimeNotSynchronized), (\"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65\", TimeNotSynchronized), (\"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a\", TimeNotSynchronized), (\"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069\", TimeNotSynchronized), (\"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451\", TimeNotSynchronized), (\"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf\", TimeNotSynchronized), (\"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300\", TimeNotSynchronized), (\"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d\", TimeNotSynchronized), (\"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8\", TimeNotSynchronized), (\"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\", TimeNotSynchronized), (\"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83\", TimeNotSynchronized), (\"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d\", TimeNotSynchronized), (\"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8\", TimeNotSynchronized)]","latency_us":485562,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:04.75258098Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"1.701845153s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"2c5a56b7-c7e9-4e2e-8b17-47af1789af0a\", \"content-length\": \"124\", \"date\": \"Mon, 05 Aug 2024 16:44:04 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"2c5a56b7-c7e9-4e2e-8b17-47af1789af0a\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:05.588616076Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","component":"MetricsManager","sled_id":"3abf3a4f-8820-40af-a01f-4369b42a5529","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"1.167683206s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 83a194e2-3464-4a84-ae47-e33a0983bc56, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: fe17df80-010a-4b68-9ebc-6a94437be875, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5b6cee62-004c-4843-9e16-c25d2b0c743b, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6300d678-c957-4259-9a3e-c167ee4d0fb0, kind: Service { id: 5b6cee62-004c-4843-9e16-c25d2b0c743b }, name: Name(\"ntp-5b6cee62-004c-4843-9e16-c25d2b0c743b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 71])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 3c470f26-8422-4139-9f7f-81a0d84becba, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: acae5a41-5da9-420f-9bc8-154e4abdaf87, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } } } }, OmicronZoneConfig { id: eed4ad2f-7765-43e1-8943-ea458ed22e57, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } } } }, OmicronZoneConfig { id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 902ea001-1913-4130-9979-87a8d3db1f53, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8995285b-5cb0-4d3a-b021-8aec05314a65, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: c8555f5b-90c8-4b5b-b73e-5b0ba7ce0695, kind: Service { id: 8995285b-5cb0-4d3a-b021-8aec05314a65 }, name: Name(\"external-dns-8995285b-5cb0-4d3a-b021-8aec05314a65\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 233, 199])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 0a07869d-c9ec-4ade-b114-97ceff54b60e, kind: Service { id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8 }, name: Name(\"external-dns-8a344ff3-4023-4f5e-a29a-3db2dc6980b8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 206, 204])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 866d6328-4ca1-435e-8a98-cd5268df29e8, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 6eedd1d7-7b63-4139-96c3-ad58e9233e7c, kind: Service { id: 866d6328-4ca1-435e-8a98-cd5268df29e8 }, name: Name(\"nexus-866d6328-4ca1-435e-8a98-cd5268df29e8\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 250])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4952ec5d-23ba-48ea-9509-011fcb02adaf, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 38978e97-2109-4173-8624-98de2b729958, kind: Service { id: 4952ec5d-23ba-48ea-9509-011fcb02adaf }, name: Name(\"nexus-4952ec5d-23ba-48ea-9509-011fcb02adaf\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 188])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1c82ddd6-6f5a-4a92-a9b3-fde01c046c42, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 51730dfc-654f-49d5-a0b7-5f1b747fb3be, kind: Service { id: 1c82ddd6-6f5a-4a92-a9b3-fde01c046c42 }, name: Name(\"nexus-1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 28])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 893afdd2-39c6-4f94-b381-facb1a40bbb1, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 061f99f8-6626-4a81-8262-abd77ceeb36a, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 15711896-d3e4-44e3-b330-624f0f5e064d, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: c02f008f-19c7-48c5-93c6-3f776c51e015, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 738351b7-dacb-486a-ab38-cf8aefaef9cb, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 29db7ccb-90e1-414b-aeee-a9393f639631, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } } } }, OmicronZoneConfig { id: d857801a-a95b-4937-a2f0-31ad7c239ac7, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External } } } }, OmicronZoneConfig { id: 58b140e5-f61e-4410-8851-af929dfd8069, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External } } } }, OmicronZoneConfig { id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External } } } }, OmicronZoneConfig { id: c1741444-7ab7-4888-aa3c-e7b6da3527da, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:06.46059426Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83a194e2-3464-4a84-ae47-e33a0983bc56, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:06.460944508Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:06.460953358Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 83a194e2-3464-4a84-ae47-e33a0983bc56, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:06.484872057Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:06.484914039Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe17df80-010a-4b68-9ebc-6a94437be875, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:06.505926012Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:06.505960776Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: fe17df80-010a-4b68-9ebc-6a94437be875, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c470f26-8422-4139-9f7f-81a0d84becba, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:06.525909789Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:06.525948481Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 3c470f26-8422-4139-9f7f-81a0d84becba, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: acae5a41-5da9-420f-9bc8-154e4abdaf87, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:06.548243009Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:06.548289735Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: acae5a41-5da9-420f-9bc8-154e4abdaf87, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eed4ad2f-7765-43e1-8943-ea458ed22e57, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:06.568461269Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:06.568503625Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: eed4ad2f-7765-43e1-8943-ea458ed22e57, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:06.590210385Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:06.590249686Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 902ea001-1913-4130-9979-87a8d3db1f53, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:06.611118992Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:06.611151872Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 902ea001-1913-4130-9979-87a8d3db1f53, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8995285b-5cb0-4d3a-b021-8aec05314a65, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:06.631776134Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:06.631828382Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8995285b-5cb0-4d3a-b021-8aec05314a65, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:06.654904221Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:06.654948069Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 061f99f8-6626-4a81-8262-abd77ceeb36a, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:06.674885425Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:06.67492569Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 061f99f8-6626-4a81-8262-abd77ceeb36a, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 29db7ccb-90e1-414b-aeee-a9393f639631, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:06.697114526Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:06.697156376Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 29db7ccb-90e1-414b-aeee-a9393f639631, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:06.719637901Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:06.719678599Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d857801a-a95b-4937-a2f0-31ad7c239ac7, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:06.742640326Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:06.742684905Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: d857801a-a95b-4937-a2f0-31ad7c239ac7, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:06.763644136Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:06.763684252Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:06.785757226Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:06.785801046Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 58b140e5-f61e-4410-8851-af929dfd8069, dataset_name: DatasetName { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:06.808020933Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:06.808059595Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 58b140e5-f61e-4410-8851-af929dfd8069, dataset_name: DatasetName { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:06.831552624Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:06.831594465Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, dataset_name: DatasetName { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:06.85298476Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:06.853024736Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, dataset_name: DatasetName { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1741444-7ab7-4888-aa3c-e7b6da3527da, dataset_name: DatasetName { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:06.875553185Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:06.875601335Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: c1741444-7ab7-4888-aa3c-e7b6da3527da, dataset_name: DatasetName { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:06.89540072Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Reading ledger from /pool/int/4576fedf-479b-43b3-8fad-bb8b7ec274f8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:06.895455949Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/d09b3f64-0a36-4052-8233-a94e62e662e2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:06.895654711Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:06.895789301Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:06.994810352Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:06.994867081Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:06.994873653Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:06.994880459Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:06.994887143Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:06.99490355Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:06.99490925Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:06.994915766Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:06.994920904Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:06.994927176Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:06.994932285Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:06.994937291Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:06.994942419Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:06.994947716Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:06.994951251Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:06.994954936Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:06.994958451Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:06.994963213Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:06.994967864Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:06.995043081Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"7880e5ec-643f-4849-8dae-1fe23ef4989d","remote_addr":"[fd00:1122:3344:101::1]:59148","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83\", TimeNotSynchronized), (\"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069\", TimeNotSynchronized), (\"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1\", TimeNotSynchronized), (\"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d\", TimeNotSynchronized), (\"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7\", TimeNotSynchronized), (\"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d\", TimeNotSynchronized), (\"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8\", TimeNotSynchronized), (\"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\", TimeNotSynchronized), (\"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf\", TimeNotSynchronized), (\"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb\", TimeNotSynchronized), (\"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631\", TimeNotSynchronized), (\"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451\", TimeNotSynchronized), (\"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300\", TimeNotSynchronized), (\"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2\", TimeNotSynchronized), (\"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da\", TimeNotSynchronized), (\"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65\", TimeNotSynchronized), (\"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8\", TimeNotSynchronized), (\"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a\", TimeNotSynchronized), (\"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015\", TimeNotSynchronized)]","latency_us":532880,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:06.995286711Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"1.592005046s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"7880e5ec-643f-4849-8dae-1fe23ef4989d\", \"content-length\": \"124\", \"date\": \"Mon, 05 Aug 2024 16:44:06 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"7880e5ec-643f-4849-8dae-1fe23ef4989d\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 83a194e2-3464-4a84-ae47-e33a0983bc56, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: fe17df80-010a-4b68-9ebc-6a94437be875, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5b6cee62-004c-4843-9e16-c25d2b0c743b, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6300d678-c957-4259-9a3e-c167ee4d0fb0, kind: Service { id: 5b6cee62-004c-4843-9e16-c25d2b0c743b }, name: Name(\"ntp-5b6cee62-004c-4843-9e16-c25d2b0c743b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 71])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 3c470f26-8422-4139-9f7f-81a0d84becba, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: acae5a41-5da9-420f-9bc8-154e4abdaf87, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } } } }, OmicronZoneConfig { id: eed4ad2f-7765-43e1-8943-ea458ed22e57, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } } } }, OmicronZoneConfig { id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 902ea001-1913-4130-9979-87a8d3db1f53, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8995285b-5cb0-4d3a-b021-8aec05314a65, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: c8555f5b-90c8-4b5b-b73e-5b0ba7ce0695, kind: Service { id: 8995285b-5cb0-4d3a-b021-8aec05314a65 }, name: Name(\"external-dns-8995285b-5cb0-4d3a-b021-8aec05314a65\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 233, 199])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 0a07869d-c9ec-4ade-b114-97ceff54b60e, kind: Service { id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8 }, name: Name(\"external-dns-8a344ff3-4023-4f5e-a29a-3db2dc6980b8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 206, 204])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 866d6328-4ca1-435e-8a98-cd5268df29e8, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 6eedd1d7-7b63-4139-96c3-ad58e9233e7c, kind: Service { id: 866d6328-4ca1-435e-8a98-cd5268df29e8 }, name: Name(\"nexus-866d6328-4ca1-435e-8a98-cd5268df29e8\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 250])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4952ec5d-23ba-48ea-9509-011fcb02adaf, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 38978e97-2109-4173-8624-98de2b729958, kind: Service { id: 4952ec5d-23ba-48ea-9509-011fcb02adaf }, name: Name(\"nexus-4952ec5d-23ba-48ea-9509-011fcb02adaf\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 188])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1c82ddd6-6f5a-4a92-a9b3-fde01c046c42, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 51730dfc-654f-49d5-a0b7-5f1b747fb3be, kind: Service { id: 1c82ddd6-6f5a-4a92-a9b3-fde01c046c42 }, name: Name(\"nexus-1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 28])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 893afdd2-39c6-4f94-b381-facb1a40bbb1, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 061f99f8-6626-4a81-8262-abd77ceeb36a, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 15711896-d3e4-44e3-b330-624f0f5e064d, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: c02f008f-19c7-48c5-93c6-3f776c51e015, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 738351b7-dacb-486a-ab38-cf8aefaef9cb, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 29db7ccb-90e1-414b-aeee-a9393f639631, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } } } }, OmicronZoneConfig { id: d857801a-a95b-4937-a2f0-31ad7c239ac7, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External } } } }, OmicronZoneConfig { id: 58b140e5-f61e-4410-8851-af929dfd8069, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External } } } }, OmicronZoneConfig { id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External } } } }, OmicronZoneConfig { id: c1741444-7ab7-4888-aa3c-e7b6da3527da, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:08.592638227Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83a194e2-3464-4a84-ae47-e33a0983bc56, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:08.593011968Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:08.593027727Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 83a194e2-3464-4a84-ae47-e33a0983bc56, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:08.614447403Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:08.614560484Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe17df80-010a-4b68-9ebc-6a94437be875, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:08.637406779Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:08.637452286Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: fe17df80-010a-4b68-9ebc-6a94437be875, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c470f26-8422-4139-9f7f-81a0d84becba, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:08.657245745Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:08.65729758Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 3c470f26-8422-4139-9f7f-81a0d84becba, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: acae5a41-5da9-420f-9bc8-154e4abdaf87, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:08.675167423Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:08.675207623Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: acae5a41-5da9-420f-9bc8-154e4abdaf87, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eed4ad2f-7765-43e1-8943-ea458ed22e57, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:08.691942989Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:08.691982542Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: eed4ad2f-7765-43e1-8943-ea458ed22e57, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:08.711636015Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:08.711675494Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 902ea001-1913-4130-9979-87a8d3db1f53, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:08.730099047Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:08.730137616Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 902ea001-1913-4130-9979-87a8d3db1f53, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8995285b-5cb0-4d3a-b021-8aec05314a65, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:08.752216274Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:08.75225371Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8995285b-5cb0-4d3a-b021-8aec05314a65, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:08.772078347Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:08.772121116Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 061f99f8-6626-4a81-8262-abd77ceeb36a, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:08.793033616Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:08.79307346Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 061f99f8-6626-4a81-8262-abd77ceeb36a, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 29db7ccb-90e1-414b-aeee-a9393f639631, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:08.815739939Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:08.815778695Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 29db7ccb-90e1-414b-aeee-a9393f639631, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:08.836800367Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:08.836838476Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d857801a-a95b-4937-a2f0-31ad7c239ac7, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:08.857005886Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:08.857045721Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: d857801a-a95b-4937-a2f0-31ad7c239ac7, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:08.877889233Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:08.877938097Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:08.897778899Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:08.897817636Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 58b140e5-f61e-4410-8851-af929dfd8069, dataset_name: DatasetName { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:08.919546235Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:08.919585628Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 58b140e5-f61e-4410-8851-af929dfd8069, dataset_name: DatasetName { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:08.94011918Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:08.940159381Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, dataset_name: DatasetName { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:08.960585042Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:08.960625008Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, dataset_name: DatasetName { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1741444-7ab7-4888-aa3c-e7b6da3527da, dataset_name: DatasetName { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:08.982150939Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:08.982191834Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: c1741444-7ab7-4888-aa3c-e7b6da3527da, dataset_name: DatasetName { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:09.002785435Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Reading ledger from /pool/int/4576fedf-479b-43b3-8fad-bb8b7ec274f8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:09.002837824Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/d09b3f64-0a36-4052-8233-a94e62e662e2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:09.003017903Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:09.003123269Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:09.104952228Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:09.105003576Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:09.105011208Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:09.10501777Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:09.105023245Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:09.105030558Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:09.105036201Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:09.105043402Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:09.105049946Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:09.105066362Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:09.105071968Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:09.105079243Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:09.105085074Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:09.105091674Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:09.105097412Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:09.105102728Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:09.105109206Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:09.105114888Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:09.105121928Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:09.105257071Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"f7f1e817-e56d-4529-8661-4bb1f7a7586e","remote_addr":"[fd00:1122:3344:101::1]:59148","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8\", TimeNotSynchronized), (\"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8\", TimeNotSynchronized), (\"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d\", TimeNotSynchronized), (\"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da\", TimeNotSynchronized), (\"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300\", TimeNotSynchronized), (\"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7\", TimeNotSynchronized), (\"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015\", TimeNotSynchronized), (\"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a\", TimeNotSynchronized), (\"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d\", TimeNotSynchronized), (\"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2\", TimeNotSynchronized), (\"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\", TimeNotSynchronized), (\"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83\", TimeNotSynchronized), (\"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf\", TimeNotSynchronized), (\"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451\", TimeNotSynchronized), (\"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1\", TimeNotSynchronized), (\"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069\", TimeNotSynchronized), (\"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631\", TimeNotSynchronized), (\"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb\", TimeNotSynchronized), (\"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65\", TimeNotSynchronized)]","latency_us":511083,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:09.105434036Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"1.24030094s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f7f1e817-e56d-4529-8661-4bb1f7a7586e\", \"content-length\": \"124\", \"date\": \"Mon, 05 Aug 2024 16:44:09 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f7f1e817-e56d-4529-8661-4bb1f7a7586e\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 83a194e2-3464-4a84-ae47-e33a0983bc56, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: fe17df80-010a-4b68-9ebc-6a94437be875, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5b6cee62-004c-4843-9e16-c25d2b0c743b, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6300d678-c957-4259-9a3e-c167ee4d0fb0, kind: Service { id: 5b6cee62-004c-4843-9e16-c25d2b0c743b }, name: Name(\"ntp-5b6cee62-004c-4843-9e16-c25d2b0c743b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 71])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 3c470f26-8422-4139-9f7f-81a0d84becba, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: acae5a41-5da9-420f-9bc8-154e4abdaf87, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } } } }, OmicronZoneConfig { id: eed4ad2f-7765-43e1-8943-ea458ed22e57, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } } } }, OmicronZoneConfig { id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 902ea001-1913-4130-9979-87a8d3db1f53, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8995285b-5cb0-4d3a-b021-8aec05314a65, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: c8555f5b-90c8-4b5b-b73e-5b0ba7ce0695, kind: Service { id: 8995285b-5cb0-4d3a-b021-8aec05314a65 }, name: Name(\"external-dns-8995285b-5cb0-4d3a-b021-8aec05314a65\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 233, 199])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 0a07869d-c9ec-4ade-b114-97ceff54b60e, kind: Service { id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8 }, name: Name(\"external-dns-8a344ff3-4023-4f5e-a29a-3db2dc6980b8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 206, 204])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 866d6328-4ca1-435e-8a98-cd5268df29e8, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 6eedd1d7-7b63-4139-96c3-ad58e9233e7c, kind: Service { id: 866d6328-4ca1-435e-8a98-cd5268df29e8 }, name: Name(\"nexus-866d6328-4ca1-435e-8a98-cd5268df29e8\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 250])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4952ec5d-23ba-48ea-9509-011fcb02adaf, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 38978e97-2109-4173-8624-98de2b729958, kind: Service { id: 4952ec5d-23ba-48ea-9509-011fcb02adaf }, name: Name(\"nexus-4952ec5d-23ba-48ea-9509-011fcb02adaf\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 188])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1c82ddd6-6f5a-4a92-a9b3-fde01c046c42, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 51730dfc-654f-49d5-a0b7-5f1b747fb3be, kind: Service { id: 1c82ddd6-6f5a-4a92-a9b3-fde01c046c42 }, name: Name(\"nexus-1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 28])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 893afdd2-39c6-4f94-b381-facb1a40bbb1, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 061f99f8-6626-4a81-8262-abd77ceeb36a, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 15711896-d3e4-44e3-b330-624f0f5e064d, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: c02f008f-19c7-48c5-93c6-3f776c51e015, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 738351b7-dacb-486a-ab38-cf8aefaef9cb, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 29db7ccb-90e1-414b-aeee-a9393f639631, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } } } }, OmicronZoneConfig { id: d857801a-a95b-4937-a2f0-31ad7c239ac7, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External } } } }, OmicronZoneConfig { id: 58b140e5-f61e-4410-8851-af929dfd8069, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External } } } }, OmicronZoneConfig { id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External } } } }, OmicronZoneConfig { id: c1741444-7ab7-4888-aa3c-e7b6da3527da, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:10.350048037Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83a194e2-3464-4a84-ae47-e33a0983bc56, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:10.350462316Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:10.350469197Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 83a194e2-3464-4a84-ae47-e33a0983bc56, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:10.373429924Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:10.373494566Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe17df80-010a-4b68-9ebc-6a94437be875, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:10.394200794Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:10.39424106Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: fe17df80-010a-4b68-9ebc-6a94437be875, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c470f26-8422-4139-9f7f-81a0d84becba, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:10.415963928Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:10.41600921Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 3c470f26-8422-4139-9f7f-81a0d84becba, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: acae5a41-5da9-420f-9bc8-154e4abdaf87, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:10.435110582Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:10.435150107Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: acae5a41-5da9-420f-9bc8-154e4abdaf87, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eed4ad2f-7765-43e1-8943-ea458ed22e57, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:10.453933987Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:10.453973259Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: eed4ad2f-7765-43e1-8943-ea458ed22e57, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:10.473460523Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:10.473576442Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 902ea001-1913-4130-9979-87a8d3db1f53, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:10.492390414Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:10.492429247Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 902ea001-1913-4130-9979-87a8d3db1f53, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8995285b-5cb0-4d3a-b021-8aec05314a65, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:10.5118743Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:10.511928975Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8995285b-5cb0-4d3a-b021-8aec05314a65, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:10.533903571Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:10.533944447Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 061f99f8-6626-4a81-8262-abd77ceeb36a, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:10.554504022Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:10.554539573Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 061f99f8-6626-4a81-8262-abd77ceeb36a, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 29db7ccb-90e1-414b-aeee-a9393f639631, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:10.576155164Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:10.576191043Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 29db7ccb-90e1-414b-aeee-a9393f639631, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:10.596422027Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:10.596460578Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d857801a-a95b-4937-a2f0-31ad7c239ac7, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:10.61712843Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:10.617165133Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: d857801a-a95b-4937-a2f0-31ad7c239ac7, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:10.639341691Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:10.639380831Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:10.660176743Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:10.660226778Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 58b140e5-f61e-4410-8851-af929dfd8069, dataset_name: DatasetName { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:10.679520439Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:10.679571786Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 58b140e5-f61e-4410-8851-af929dfd8069, dataset_name: DatasetName { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:10.700528091Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:10.700565714Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, dataset_name: DatasetName { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:10.720868429Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:10.720912408Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, dataset_name: DatasetName { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1741444-7ab7-4888-aa3c-e7b6da3527da, dataset_name: DatasetName { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:10.740062879Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:10.740103136Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: c1741444-7ab7-4888-aa3c-e7b6da3527da, dataset_name: DatasetName { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:10.76263046Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Reading ledger from /pool/int/4576fedf-479b-43b3-8fad-bb8b7ec274f8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:10.762701739Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/d09b3f64-0a36-4052-8233-a94e62e662e2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:10.762846706Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:10.762945342Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:10.866288683Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:10.866354731Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:10.866362831Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:10.866368081Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:10.866373856Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:10.866380475Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:10.86638536Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:10.866390648Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:10.866399179Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:10.866406023Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:10.866410195Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:10.866415642Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:10.866431551Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:10.86643728Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:10.866442464Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:10.866447546Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:10.866454155Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:10.866459377Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:10.866464889Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:10.866561154Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"e56ab5a0-7acd-443a-9887-725b54131dce","remote_addr":"[fd00:1122:3344:101::1]:59148","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2\", TimeNotSynchronized), (\"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069\", TimeNotSynchronized), (\"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451\", TimeNotSynchronized), (\"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8\", TimeNotSynchronized), (\"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da\", TimeNotSynchronized), (\"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300\", TimeNotSynchronized), (\"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb\", TimeNotSynchronized), (\"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d\", TimeNotSynchronized), (\"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65\", TimeNotSynchronized), (\"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8\", TimeNotSynchronized), (\"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf\", TimeNotSynchronized), (\"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\", TimeNotSynchronized), (\"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1\", TimeNotSynchronized), (\"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d\", TimeNotSynchronized), (\"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015\", TimeNotSynchronized), (\"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631\", TimeNotSynchronized), (\"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a\", TimeNotSynchronized), (\"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83\", TimeNotSynchronized), (\"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7\", TimeNotSynchronized)]","latency_us":514996,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:10.866791311Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"1.574239034s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"e56ab5a0-7acd-443a-9887-725b54131dce\", \"content-length\": \"124\", \"date\": \"Mon, 05 Aug 2024 16:44:10 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"e56ab5a0-7acd-443a-9887-725b54131dce\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 83a194e2-3464-4a84-ae47-e33a0983bc56, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: fe17df80-010a-4b68-9ebc-6a94437be875, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5b6cee62-004c-4843-9e16-c25d2b0c743b, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6300d678-c957-4259-9a3e-c167ee4d0fb0, kind: Service { id: 5b6cee62-004c-4843-9e16-c25d2b0c743b }, name: Name(\"ntp-5b6cee62-004c-4843-9e16-c25d2b0c743b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 71])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 3c470f26-8422-4139-9f7f-81a0d84becba, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: acae5a41-5da9-420f-9bc8-154e4abdaf87, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } } } }, OmicronZoneConfig { id: eed4ad2f-7765-43e1-8943-ea458ed22e57, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } } } }, OmicronZoneConfig { id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 902ea001-1913-4130-9979-87a8d3db1f53, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8995285b-5cb0-4d3a-b021-8aec05314a65, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: c8555f5b-90c8-4b5b-b73e-5b0ba7ce0695, kind: Service { id: 8995285b-5cb0-4d3a-b021-8aec05314a65 }, name: Name(\"external-dns-8995285b-5cb0-4d3a-b021-8aec05314a65\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 233, 199])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 0a07869d-c9ec-4ade-b114-97ceff54b60e, kind: Service { id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8 }, name: Name(\"external-dns-8a344ff3-4023-4f5e-a29a-3db2dc6980b8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 206, 204])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 866d6328-4ca1-435e-8a98-cd5268df29e8, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 6eedd1d7-7b63-4139-96c3-ad58e9233e7c, kind: Service { id: 866d6328-4ca1-435e-8a98-cd5268df29e8 }, name: Name(\"nexus-866d6328-4ca1-435e-8a98-cd5268df29e8\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 250])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4952ec5d-23ba-48ea-9509-011fcb02adaf, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 38978e97-2109-4173-8624-98de2b729958, kind: Service { id: 4952ec5d-23ba-48ea-9509-011fcb02adaf }, name: Name(\"nexus-4952ec5d-23ba-48ea-9509-011fcb02adaf\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 188])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1c82ddd6-6f5a-4a92-a9b3-fde01c046c42, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 51730dfc-654f-49d5-a0b7-5f1b747fb3be, kind: Service { id: 1c82ddd6-6f5a-4a92-a9b3-fde01c046c42 }, name: Name(\"nexus-1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 28])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 893afdd2-39c6-4f94-b381-facb1a40bbb1, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 061f99f8-6626-4a81-8262-abd77ceeb36a, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 15711896-d3e4-44e3-b330-624f0f5e064d, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: c02f008f-19c7-48c5-93c6-3f776c51e015, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 738351b7-dacb-486a-ab38-cf8aefaef9cb, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 29db7ccb-90e1-414b-aeee-a9393f639631, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } } } }, OmicronZoneConfig { id: d857801a-a95b-4937-a2f0-31ad7c239ac7, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External } } } }, OmicronZoneConfig { id: 58b140e5-f61e-4410-8851-af929dfd8069, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External } } } }, OmicronZoneConfig { id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External } } } }, OmicronZoneConfig { id: c1741444-7ab7-4888-aa3c-e7b6da3527da, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:12.446782742Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83a194e2-3464-4a84-ae47-e33a0983bc56, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:12.447784595Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:12.447791608Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 83a194e2-3464-4a84-ae47-e33a0983bc56, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:12.469320203Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:12.469365428Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe17df80-010a-4b68-9ebc-6a94437be875, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:12.488913672Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:12.488952465Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: fe17df80-010a-4b68-9ebc-6a94437be875, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c470f26-8422-4139-9f7f-81a0d84becba, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:12.509279553Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:12.509317607Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 3c470f26-8422-4139-9f7f-81a0d84becba, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: acae5a41-5da9-420f-9bc8-154e4abdaf87, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:12.527886476Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:12.52793053Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: acae5a41-5da9-420f-9bc8-154e4abdaf87, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eed4ad2f-7765-43e1-8943-ea458ed22e57, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:12.546392995Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:12.546431209Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: eed4ad2f-7765-43e1-8943-ea458ed22e57, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:12.565152765Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:12.565254494Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 902ea001-1913-4130-9979-87a8d3db1f53, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:12.583535476Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:12.583574936Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 902ea001-1913-4130-9979-87a8d3db1f53, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8995285b-5cb0-4d3a-b021-8aec05314a65, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:12.603925915Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:12.603969922Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8995285b-5cb0-4d3a-b021-8aec05314a65, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:12.625153335Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:12.625203988Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 061f99f8-6626-4a81-8262-abd77ceeb36a, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:12.646232954Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:12.646278237Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 061f99f8-6626-4a81-8262-abd77ceeb36a, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 29db7ccb-90e1-414b-aeee-a9393f639631, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:12.666078517Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:12.666117677Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 29db7ccb-90e1-414b-aeee-a9393f639631, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:12.686357452Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:12.686398064Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d857801a-a95b-4937-a2f0-31ad7c239ac7, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:12.708051498Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:12.708139906Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: d857801a-a95b-4937-a2f0-31ad7c239ac7, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:12.734184087Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:12.73423729Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:12.757840559Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:12.75788496Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 58b140e5-f61e-4410-8851-af929dfd8069, dataset_name: DatasetName { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:12.7796371Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:12.779681407Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 58b140e5-f61e-4410-8851-af929dfd8069, dataset_name: DatasetName { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:12.801673395Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:12.801716942Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, dataset_name: DatasetName { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:12.822811359Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:12.822848653Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, dataset_name: DatasetName { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1741444-7ab7-4888-aa3c-e7b6da3527da, dataset_name: DatasetName { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:12.84555752Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:12.845606177Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: c1741444-7ab7-4888-aa3c-e7b6da3527da, dataset_name: DatasetName { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:12.866349162Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Reading ledger from /pool/int/4576fedf-479b-43b3-8fad-bb8b7ec274f8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:12.866397782Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/d09b3f64-0a36-4052-8233-a94e62e662e2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:12.86660217Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:12.866727947Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:12.964175883Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:12.964235359Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:12.964241697Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:12.964248606Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:12.964254972Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:12.964260325Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:12.964333094Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:12.964338334Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:12.96434334Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:12.964349903Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:12.96435656Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:12.96436182Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:12.964368682Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:12.964375582Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:12.964382257Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:12.964387441Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:12.964395466Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:12.964399207Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:12.96440426Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:12.964467525Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"3022681b-17a9-4f9e-a324-40a4ad9e5d7f","remote_addr":"[fd00:1122:3344:101::1]:59148","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8\", TimeNotSynchronized), (\"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65\", TimeNotSynchronized), (\"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7\", TimeNotSynchronized), (\"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451\", TimeNotSynchronized), (\"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf\", TimeNotSynchronized), (\"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da\", TimeNotSynchronized), (\"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83\", TimeNotSynchronized), (\"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2\", TimeNotSynchronized), (\"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1\", TimeNotSynchronized), (\"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069\", TimeNotSynchronized), (\"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a\", TimeNotSynchronized), (\"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb\", TimeNotSynchronized), (\"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d\", TimeNotSynchronized), (\"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631\", TimeNotSynchronized), (\"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300\", TimeNotSynchronized), (\"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8\", TimeNotSynchronized), (\"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d\", TimeNotSynchronized), (\"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\", TimeNotSynchronized), (\"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015\", TimeNotSynchronized)]","latency_us":516005,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:12.964624258Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"1.988941976s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"3022681b-17a9-4f9e-a324-40a4ad9e5d7f\", \"content-length\": \"124\", \"date\": \"Mon, 05 Aug 2024 16:44:12 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"3022681b-17a9-4f9e-a324-40a4ad9e5d7f\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 83a194e2-3464-4a84-ae47-e33a0983bc56, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: fe17df80-010a-4b68-9ebc-6a94437be875, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5b6cee62-004c-4843-9e16-c25d2b0c743b, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6300d678-c957-4259-9a3e-c167ee4d0fb0, kind: Service { id: 5b6cee62-004c-4843-9e16-c25d2b0c743b }, name: Name(\"ntp-5b6cee62-004c-4843-9e16-c25d2b0c743b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 71])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 3c470f26-8422-4139-9f7f-81a0d84becba, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: acae5a41-5da9-420f-9bc8-154e4abdaf87, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } } } }, OmicronZoneConfig { id: eed4ad2f-7765-43e1-8943-ea458ed22e57, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } } } }, OmicronZoneConfig { id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 902ea001-1913-4130-9979-87a8d3db1f53, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8995285b-5cb0-4d3a-b021-8aec05314a65, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: c8555f5b-90c8-4b5b-b73e-5b0ba7ce0695, kind: Service { id: 8995285b-5cb0-4d3a-b021-8aec05314a65 }, name: Name(\"external-dns-8995285b-5cb0-4d3a-b021-8aec05314a65\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 233, 199])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 0a07869d-c9ec-4ade-b114-97ceff54b60e, kind: Service { id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8 }, name: Name(\"external-dns-8a344ff3-4023-4f5e-a29a-3db2dc6980b8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 206, 204])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 866d6328-4ca1-435e-8a98-cd5268df29e8, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 6eedd1d7-7b63-4139-96c3-ad58e9233e7c, kind: Service { id: 866d6328-4ca1-435e-8a98-cd5268df29e8 }, name: Name(\"nexus-866d6328-4ca1-435e-8a98-cd5268df29e8\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 250])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4952ec5d-23ba-48ea-9509-011fcb02adaf, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 38978e97-2109-4173-8624-98de2b729958, kind: Service { id: 4952ec5d-23ba-48ea-9509-011fcb02adaf }, name: Name(\"nexus-4952ec5d-23ba-48ea-9509-011fcb02adaf\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 188])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1c82ddd6-6f5a-4a92-a9b3-fde01c046c42, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 51730dfc-654f-49d5-a0b7-5f1b747fb3be, kind: Service { id: 1c82ddd6-6f5a-4a92-a9b3-fde01c046c42 }, name: Name(\"nexus-1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 28])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 893afdd2-39c6-4f94-b381-facb1a40bbb1, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 061f99f8-6626-4a81-8262-abd77ceeb36a, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 15711896-d3e4-44e3-b330-624f0f5e064d, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: c02f008f-19c7-48c5-93c6-3f776c51e015, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 738351b7-dacb-486a-ab38-cf8aefaef9cb, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 29db7ccb-90e1-414b-aeee-a9393f639631, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } } } }, OmicronZoneConfig { id: d857801a-a95b-4937-a2f0-31ad7c239ac7, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External } } } }, OmicronZoneConfig { id: 58b140e5-f61e-4410-8851-af929dfd8069, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External } } } }, OmicronZoneConfig { id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External } } } }, OmicronZoneConfig { id: c1741444-7ab7-4888-aa3c-e7b6da3527da, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:14.961109811Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83a194e2-3464-4a84-ae47-e33a0983bc56, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:14.961633395Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:14.961649098Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 83a194e2-3464-4a84-ae47-e33a0983bc56, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:14.984166484Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:14.984207322Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe17df80-010a-4b68-9ebc-6a94437be875, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:15.005519259Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:15.005552532Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: fe17df80-010a-4b68-9ebc-6a94437be875, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c470f26-8422-4139-9f7f-81a0d84becba, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:15.026689785Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:15.026734636Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 3c470f26-8422-4139-9f7f-81a0d84becba, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: acae5a41-5da9-420f-9bc8-154e4abdaf87, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:15.047133032Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:15.047173354Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: acae5a41-5da9-420f-9bc8-154e4abdaf87, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eed4ad2f-7765-43e1-8943-ea458ed22e57, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:15.067631351Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:15.06766617Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: eed4ad2f-7765-43e1-8943-ea458ed22e57, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:15.086198613Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:15.086233676Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 902ea001-1913-4130-9979-87a8d3db1f53, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:15.106542272Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:15.10657456Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 902ea001-1913-4130-9979-87a8d3db1f53, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8995285b-5cb0-4d3a-b021-8aec05314a65, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:15.124348597Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:15.124406536Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8995285b-5cb0-4d3a-b021-8aec05314a65, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:15.143868011Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:15.143917465Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 061f99f8-6626-4a81-8262-abd77ceeb36a, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:15.162369949Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:15.162404918Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 061f99f8-6626-4a81-8262-abd77ceeb36a, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 29db7ccb-90e1-414b-aeee-a9393f639631, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:15.180444856Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:15.180483792Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 29db7ccb-90e1-414b-aeee-a9393f639631, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:15.19890322Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:15.198934364Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d857801a-a95b-4937-a2f0-31ad7c239ac7, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:15.217114577Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:15.217165052Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: d857801a-a95b-4937-a2f0-31ad7c239ac7, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:15.235639289Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:15.235683662Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:15.25377959Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:15.253820991Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 58b140e5-f61e-4410-8851-af929dfd8069, dataset_name: DatasetName { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:15.271969971Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:15.272003881Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 58b140e5-f61e-4410-8851-af929dfd8069, dataset_name: DatasetName { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:15.291840625Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:15.291873148Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, dataset_name: DatasetName { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:15.310776163Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:15.310813768Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, dataset_name: DatasetName { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1741444-7ab7-4888-aa3c-e7b6da3527da, dataset_name: DatasetName { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:15.328940292Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:15.328995408Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: c1741444-7ab7-4888-aa3c-e7b6da3527da, dataset_name: DatasetName { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:15.348039709Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Reading ledger from /pool/int/4576fedf-479b-43b3-8fad-bb8b7ec274f8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:15.348082506Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/d09b3f64-0a36-4052-8233-a94e62e662e2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:15.348302091Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:15.348425412Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:15.440746123Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:15.440786615Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:15.440795296Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:15.440800565Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:15.440807099Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:15.440824341Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:15.440830988Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:15.440836388Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:15.440841319Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:15.440846372Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:15.440851631Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:15.440858109Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:15.440864372Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:15.440870738Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:15.440875492Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:15.440881856Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:15.44088692Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:15.440893313Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:15.440899942Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:15.44105746Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"2feea260-6138-40dc-a7c8-f9e9088013ff","remote_addr":"[fd00:1122:3344:101::1]:59148","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83\", TimeNotSynchronized), (\"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015\", TimeNotSynchronized), (\"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d\", TimeNotSynchronized), (\"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8\", TimeNotSynchronized), (\"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\", TimeNotSynchronized), (\"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631\", TimeNotSynchronized), (\"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb\", TimeNotSynchronized), (\"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf\", TimeNotSynchronized), (\"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8\", TimeNotSynchronized), (\"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1\", TimeNotSynchronized), (\"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65\", TimeNotSynchronized), (\"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2\", TimeNotSynchronized), (\"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d\", TimeNotSynchronized), (\"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069\", TimeNotSynchronized), (\"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451\", TimeNotSynchronized), (\"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a\", TimeNotSynchronized), (\"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300\", TimeNotSynchronized), (\"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da\", TimeNotSynchronized), (\"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7\", TimeNotSynchronized)]","latency_us":478418,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:15.441203506Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"1.952355604s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"2feea260-6138-40dc-a7c8-f9e9088013ff\", \"content-length\": \"124\", \"date\": \"Mon, 05 Aug 2024 16:44:14 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"2feea260-6138-40dc-a7c8-f9e9088013ff\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 83a194e2-3464-4a84-ae47-e33a0983bc56, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: fe17df80-010a-4b68-9ebc-6a94437be875, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5b6cee62-004c-4843-9e16-c25d2b0c743b, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6300d678-c957-4259-9a3e-c167ee4d0fb0, kind: Service { id: 5b6cee62-004c-4843-9e16-c25d2b0c743b }, name: Name(\"ntp-5b6cee62-004c-4843-9e16-c25d2b0c743b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 71])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 3c470f26-8422-4139-9f7f-81a0d84becba, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: acae5a41-5da9-420f-9bc8-154e4abdaf87, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } } } }, OmicronZoneConfig { id: eed4ad2f-7765-43e1-8943-ea458ed22e57, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } } } }, OmicronZoneConfig { id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 902ea001-1913-4130-9979-87a8d3db1f53, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8995285b-5cb0-4d3a-b021-8aec05314a65, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: c8555f5b-90c8-4b5b-b73e-5b0ba7ce0695, kind: Service { id: 8995285b-5cb0-4d3a-b021-8aec05314a65 }, name: Name(\"external-dns-8995285b-5cb0-4d3a-b021-8aec05314a65\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 233, 199])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 0a07869d-c9ec-4ade-b114-97ceff54b60e, kind: Service { id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8 }, name: Name(\"external-dns-8a344ff3-4023-4f5e-a29a-3db2dc6980b8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 206, 204])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 866d6328-4ca1-435e-8a98-cd5268df29e8, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 6eedd1d7-7b63-4139-96c3-ad58e9233e7c, kind: Service { id: 866d6328-4ca1-435e-8a98-cd5268df29e8 }, name: Name(\"nexus-866d6328-4ca1-435e-8a98-cd5268df29e8\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 250])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4952ec5d-23ba-48ea-9509-011fcb02adaf, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 38978e97-2109-4173-8624-98de2b729958, kind: Service { id: 4952ec5d-23ba-48ea-9509-011fcb02adaf }, name: Name(\"nexus-4952ec5d-23ba-48ea-9509-011fcb02adaf\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 188])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1c82ddd6-6f5a-4a92-a9b3-fde01c046c42, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 51730dfc-654f-49d5-a0b7-5f1b747fb3be, kind: Service { id: 1c82ddd6-6f5a-4a92-a9b3-fde01c046c42 }, name: Name(\"nexus-1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 28])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 893afdd2-39c6-4f94-b381-facb1a40bbb1, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 061f99f8-6626-4a81-8262-abd77ceeb36a, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 15711896-d3e4-44e3-b330-624f0f5e064d, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: c02f008f-19c7-48c5-93c6-3f776c51e015, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 738351b7-dacb-486a-ab38-cf8aefaef9cb, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 29db7ccb-90e1-414b-aeee-a9393f639631, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } } } }, OmicronZoneConfig { id: d857801a-a95b-4937-a2f0-31ad7c239ac7, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External } } } }, OmicronZoneConfig { id: 58b140e5-f61e-4410-8851-af929dfd8069, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External } } } }, OmicronZoneConfig { id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External } } } }, OmicronZoneConfig { id: c1741444-7ab7-4888-aa3c-e7b6da3527da, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:17.400621363Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83a194e2-3464-4a84-ae47-e33a0983bc56, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:17.401024963Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:17.40103671Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 83a194e2-3464-4a84-ae47-e33a0983bc56, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:17.424457521Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:17.424494956Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe17df80-010a-4b68-9ebc-6a94437be875, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:17.444076956Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:17.444105194Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: fe17df80-010a-4b68-9ebc-6a94437be875, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c470f26-8422-4139-9f7f-81a0d84becba, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:17.463112571Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:17.463146518Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 3c470f26-8422-4139-9f7f-81a0d84becba, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: acae5a41-5da9-420f-9bc8-154e4abdaf87, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:17.48192171Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:17.481963786Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: acae5a41-5da9-420f-9bc8-154e4abdaf87, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eed4ad2f-7765-43e1-8943-ea458ed22e57, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:17.502039018Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:17.502078956Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: eed4ad2f-7765-43e1-8943-ea458ed22e57, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:17.520840528Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:17.520880682Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 902ea001-1913-4130-9979-87a8d3db1f53, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:17.539920837Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:17.539960597Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 902ea001-1913-4130-9979-87a8d3db1f53, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8995285b-5cb0-4d3a-b021-8aec05314a65, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:17.559340428Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:17.55938781Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8995285b-5cb0-4d3a-b021-8aec05314a65, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:17.578383659Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:17.578424881Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 061f99f8-6626-4a81-8262-abd77ceeb36a, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:17.598294098Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:17.598332123Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 061f99f8-6626-4a81-8262-abd77ceeb36a, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 29db7ccb-90e1-414b-aeee-a9393f639631, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:17.616525831Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:17.616573476Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 29db7ccb-90e1-414b-aeee-a9393f639631, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:17.636077956Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:17.636113085Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d857801a-a95b-4937-a2f0-31ad7c239ac7, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:17.654273583Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:17.654309424Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: d857801a-a95b-4937-a2f0-31ad7c239ac7, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:17.672475449Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:17.672535273Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:17.691635229Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:17.691671774Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 58b140e5-f61e-4410-8851-af929dfd8069, dataset_name: DatasetName { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:17.711625734Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:17.713977431Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 58b140e5-f61e-4410-8851-af929dfd8069, dataset_name: DatasetName { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:17.732276111Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:17.73232188Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, dataset_name: DatasetName { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:17.751576788Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:17.751614149Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, dataset_name: DatasetName { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1741444-7ab7-4888-aa3c-e7b6da3527da, dataset_name: DatasetName { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:17.770541518Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:17.770582609Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: c1741444-7ab7-4888-aa3c-e7b6da3527da, dataset_name: DatasetName { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:17.789577889Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Reading ledger from /pool/int/4576fedf-479b-43b3-8fad-bb8b7ec274f8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:17.789631205Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/d09b3f64-0a36-4052-8233-a94e62e662e2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:17.789808433Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:17.789912188Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:17.883278867Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:17.883342534Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:17.883352246Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:17.883359812Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:17.883371644Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:17.883378647Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:17.883385453Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:17.883389963Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:17.8833978Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:17.883423807Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:17.883432451Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:17.883446251Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:17.88345137Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:17.883458017Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:17.883464757Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:17.883472792Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:17.883479448Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:17.88348481Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:17.883498377Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:17.883535399Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"392529e2-3544-4754-83b4-745867e623c5","remote_addr":"[fd00:1122:3344:101::1]:59148","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8\", TimeNotSynchronized), (\"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d\", TimeNotSynchronized), (\"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da\", TimeNotSynchronized), (\"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf\", TimeNotSynchronized), (\"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65\", TimeNotSynchronized), (\"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451\", TimeNotSynchronized), (\"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83\", TimeNotSynchronized), (\"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\", TimeNotSynchronized), (\"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069\", TimeNotSynchronized), (\"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300\", TimeNotSynchronized), (\"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8\", TimeNotSynchronized), (\"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631\", TimeNotSynchronized), (\"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2\", TimeNotSynchronized), (\"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015\", TimeNotSynchronized), (\"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d\", TimeNotSynchronized), (\"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1\", TimeNotSynchronized), (\"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb\", TimeNotSynchronized), (\"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a\", TimeNotSynchronized), (\"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7\", TimeNotSynchronized)]","latency_us":481423,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:17.883764292Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"2.412328217s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"392529e2-3544-4754-83b4-745867e623c5\", \"content-length\": \"124\", \"date\": \"Mon, 05 Aug 2024 16:44:17 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"392529e2-3544-4754-83b4-745867e623c5\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 83a194e2-3464-4a84-ae47-e33a0983bc56, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: fe17df80-010a-4b68-9ebc-6a94437be875, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5b6cee62-004c-4843-9e16-c25d2b0c743b, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6300d678-c957-4259-9a3e-c167ee4d0fb0, kind: Service { id: 5b6cee62-004c-4843-9e16-c25d2b0c743b }, name: Name(\"ntp-5b6cee62-004c-4843-9e16-c25d2b0c743b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 71])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 3c470f26-8422-4139-9f7f-81a0d84becba, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: acae5a41-5da9-420f-9bc8-154e4abdaf87, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } } } }, OmicronZoneConfig { id: eed4ad2f-7765-43e1-8943-ea458ed22e57, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } } } }, OmicronZoneConfig { id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 902ea001-1913-4130-9979-87a8d3db1f53, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8995285b-5cb0-4d3a-b021-8aec05314a65, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: c8555f5b-90c8-4b5b-b73e-5b0ba7ce0695, kind: Service { id: 8995285b-5cb0-4d3a-b021-8aec05314a65 }, name: Name(\"external-dns-8995285b-5cb0-4d3a-b021-8aec05314a65\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 233, 199])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 0a07869d-c9ec-4ade-b114-97ceff54b60e, kind: Service { id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8 }, name: Name(\"external-dns-8a344ff3-4023-4f5e-a29a-3db2dc6980b8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 206, 204])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 866d6328-4ca1-435e-8a98-cd5268df29e8, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 6eedd1d7-7b63-4139-96c3-ad58e9233e7c, kind: Service { id: 866d6328-4ca1-435e-8a98-cd5268df29e8 }, name: Name(\"nexus-866d6328-4ca1-435e-8a98-cd5268df29e8\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 250])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4952ec5d-23ba-48ea-9509-011fcb02adaf, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 38978e97-2109-4173-8624-98de2b729958, kind: Service { id: 4952ec5d-23ba-48ea-9509-011fcb02adaf }, name: Name(\"nexus-4952ec5d-23ba-48ea-9509-011fcb02adaf\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 188])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1c82ddd6-6f5a-4a92-a9b3-fde01c046c42, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 51730dfc-654f-49d5-a0b7-5f1b747fb3be, kind: Service { id: 1c82ddd6-6f5a-4a92-a9b3-fde01c046c42 }, name: Name(\"nexus-1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 28])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 893afdd2-39c6-4f94-b381-facb1a40bbb1, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 061f99f8-6626-4a81-8262-abd77ceeb36a, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 15711896-d3e4-44e3-b330-624f0f5e064d, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: c02f008f-19c7-48c5-93c6-3f776c51e015, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 738351b7-dacb-486a-ab38-cf8aefaef9cb, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 29db7ccb-90e1-414b-aeee-a9393f639631, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } } } }, OmicronZoneConfig { id: d857801a-a95b-4937-a2f0-31ad7c239ac7, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External } } } }, OmicronZoneConfig { id: 58b140e5-f61e-4410-8851-af929dfd8069, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External } } } }, OmicronZoneConfig { id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External } } } }, OmicronZoneConfig { id: c1741444-7ab7-4888-aa3c-e7b6da3527da, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:20.303983351Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83a194e2-3464-4a84-ae47-e33a0983bc56, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:20.304513707Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:20.304527301Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 83a194e2-3464-4a84-ae47-e33a0983bc56, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:20.327396534Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:20.329719545Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe17df80-010a-4b68-9ebc-6a94437be875, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:20.346608873Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:20.348620333Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: fe17df80-010a-4b68-9ebc-6a94437be875, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c470f26-8422-4139-9f7f-81a0d84becba, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:20.367526165Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:20.367578282Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 3c470f26-8422-4139-9f7f-81a0d84becba, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: acae5a41-5da9-420f-9bc8-154e4abdaf87, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:20.385911789Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:20.38595454Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: acae5a41-5da9-420f-9bc8-154e4abdaf87, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eed4ad2f-7765-43e1-8943-ea458ed22e57, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:20.404177114Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:20.404210284Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: eed4ad2f-7765-43e1-8943-ea458ed22e57, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:20.422917434Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:20.422985563Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 902ea001-1913-4130-9979-87a8d3db1f53, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:20.441650095Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:20.441686105Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 902ea001-1913-4130-9979-87a8d3db1f53, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8995285b-5cb0-4d3a-b021-8aec05314a65, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:20.460105946Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:20.460147507Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8995285b-5cb0-4d3a-b021-8aec05314a65, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:20.479204134Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:20.47924949Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 061f99f8-6626-4a81-8262-abd77ceeb36a, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:20.499037414Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:20.499072523Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 061f99f8-6626-4a81-8262-abd77ceeb36a, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 29db7ccb-90e1-414b-aeee-a9393f639631, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:20.518220624Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:20.518262287Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 29db7ccb-90e1-414b-aeee-a9393f639631, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:20.536606613Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:20.536654811Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d857801a-a95b-4937-a2f0-31ad7c239ac7, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:20.555362547Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:20.555411542Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: d857801a-a95b-4937-a2f0-31ad7c239ac7, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:20.573444938Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:20.573483883Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:20.59208446Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:20.594418874Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 58b140e5-f61e-4410-8851-af929dfd8069, dataset_name: DatasetName { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:20.611477142Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:20.611518Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 58b140e5-f61e-4410-8851-af929dfd8069, dataset_name: DatasetName { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:20.629452242Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:20.629497749Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, dataset_name: DatasetName { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:20.64971159Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:20.649759835Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, dataset_name: DatasetName { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1741444-7ab7-4888-aa3c-e7b6da3527da, dataset_name: DatasetName { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:20.668135625Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:20.668245408Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: c1741444-7ab7-4888-aa3c-e7b6da3527da, dataset_name: DatasetName { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:20.686832634Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Reading ledger from /pool/int/4576fedf-479b-43b3-8fad-bb8b7ec274f8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:20.686899357Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/d09b3f64-0a36-4052-8233-a94e62e662e2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:20.687085314Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:20.687217381Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:20.780158112Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:20.780208981Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:20.780216614Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:20.780231323Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:20.780238495Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:20.780244607Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:20.780250354Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:20.780255895Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:20.780263095Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:20.780267239Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:20.78028628Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:20.78029333Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:20.780298571Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:20.780323302Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:20.780328666Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:20.780335781Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:20.780341134Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:20.780346487Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:20.780350434Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:20.780419782Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"b6864bd3-95d2-40fe-98ba-083f9cc364f3","remote_addr":"[fd00:1122:3344:101::1]:59148","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015\", TimeNotSynchronized), (\"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2\", TimeNotSynchronized), (\"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\", TimeNotSynchronized), (\"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1\", TimeNotSynchronized), (\"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8\", TimeNotSynchronized), (\"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65\", TimeNotSynchronized), (\"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631\", TimeNotSynchronized), (\"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83\", TimeNotSynchronized), (\"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7\", TimeNotSynchronized), (\"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8\", TimeNotSynchronized), (\"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a\", TimeNotSynchronized), (\"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d\", TimeNotSynchronized), (\"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069\", TimeNotSynchronized), (\"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300\", TimeNotSynchronized), (\"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451\", TimeNotSynchronized), (\"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da\", TimeNotSynchronized), (\"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf\", TimeNotSynchronized), (\"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb\", TimeNotSynchronized), (\"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d\", TimeNotSynchronized)]","latency_us":475007,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:20.780637024Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"3.14396487s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"b6864bd3-95d2-40fe-98ba-083f9cc364f3\", \"content-length\": \"124\", \"date\": \"Mon, 05 Aug 2024 16:44:20 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"b6864bd3-95d2-40fe-98ba-083f9cc364f3\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:21.858085001Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","component":"MetricsManager","sled_id":"3abf3a4f-8820-40af-a01f-4369b42a5529","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"3.698608483s"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:23.428293008Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:60189"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:23.428393651Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"f6fe5078-a6af-4cea-9744-90bd101ac1fd","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:60189","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":44,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 83a194e2-3464-4a84-ae47-e33a0983bc56, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: fe17df80-010a-4b68-9ebc-6a94437be875, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5b6cee62-004c-4843-9e16-c25d2b0c743b, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6300d678-c957-4259-9a3e-c167ee4d0fb0, kind: Service { id: 5b6cee62-004c-4843-9e16-c25d2b0c743b }, name: Name(\"ntp-5b6cee62-004c-4843-9e16-c25d2b0c743b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 71])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 3c470f26-8422-4139-9f7f-81a0d84becba, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: acae5a41-5da9-420f-9bc8-154e4abdaf87, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } } } }, OmicronZoneConfig { id: eed4ad2f-7765-43e1-8943-ea458ed22e57, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } } } }, OmicronZoneConfig { id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 902ea001-1913-4130-9979-87a8d3db1f53, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8995285b-5cb0-4d3a-b021-8aec05314a65, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: c8555f5b-90c8-4b5b-b73e-5b0ba7ce0695, kind: Service { id: 8995285b-5cb0-4d3a-b021-8aec05314a65 }, name: Name(\"external-dns-8995285b-5cb0-4d3a-b021-8aec05314a65\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 233, 199])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 0a07869d-c9ec-4ade-b114-97ceff54b60e, kind: Service { id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8 }, name: Name(\"external-dns-8a344ff3-4023-4f5e-a29a-3db2dc6980b8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 206, 204])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 866d6328-4ca1-435e-8a98-cd5268df29e8, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 6eedd1d7-7b63-4139-96c3-ad58e9233e7c, kind: Service { id: 866d6328-4ca1-435e-8a98-cd5268df29e8 }, name: Name(\"nexus-866d6328-4ca1-435e-8a98-cd5268df29e8\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 250])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4952ec5d-23ba-48ea-9509-011fcb02adaf, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 38978e97-2109-4173-8624-98de2b729958, kind: Service { id: 4952ec5d-23ba-48ea-9509-011fcb02adaf }, name: Name(\"nexus-4952ec5d-23ba-48ea-9509-011fcb02adaf\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 188])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1c82ddd6-6f5a-4a92-a9b3-fde01c046c42, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 51730dfc-654f-49d5-a0b7-5f1b747fb3be, kind: Service { id: 1c82ddd6-6f5a-4a92-a9b3-fde01c046c42 }, name: Name(\"nexus-1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 28])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 893afdd2-39c6-4f94-b381-facb1a40bbb1, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 061f99f8-6626-4a81-8262-abd77ceeb36a, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 15711896-d3e4-44e3-b330-624f0f5e064d, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: c02f008f-19c7-48c5-93c6-3f776c51e015, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 738351b7-dacb-486a-ab38-cf8aefaef9cb, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 29db7ccb-90e1-414b-aeee-a9393f639631, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } } } }, OmicronZoneConfig { id: d857801a-a95b-4937-a2f0-31ad7c239ac7, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External } } } }, OmicronZoneConfig { id: 58b140e5-f61e-4410-8851-af929dfd8069, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External } } } }, OmicronZoneConfig { id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External } } } }, OmicronZoneConfig { id: c1741444-7ab7-4888-aa3c-e7b6da3527da, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:23.933746826Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83a194e2-3464-4a84-ae47-e33a0983bc56, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:23.934274112Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:23.934286431Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 83a194e2-3464-4a84-ae47-e33a0983bc56, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:23.957560402Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:23.957626788Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe17df80-010a-4b68-9ebc-6a94437be875, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:23.978149513Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:23.978267532Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: fe17df80-010a-4b68-9ebc-6a94437be875, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c470f26-8422-4139-9f7f-81a0d84becba, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:23.999035758Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:23.999083881Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 3c470f26-8422-4139-9f7f-81a0d84becba, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: acae5a41-5da9-420f-9bc8-154e4abdaf87, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:24.02263375Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:24.022693216Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: acae5a41-5da9-420f-9bc8-154e4abdaf87, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eed4ad2f-7765-43e1-8943-ea458ed22e57, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:24.042878274Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:24.04293835Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: eed4ad2f-7765-43e1-8943-ea458ed22e57, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:24.061241057Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:24.061290708Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 902ea001-1913-4130-9979-87a8d3db1f53, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:24.080693125Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:24.080740666Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 902ea001-1913-4130-9979-87a8d3db1f53, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8995285b-5cb0-4d3a-b021-8aec05314a65, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:24.099765446Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:24.099820094Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8995285b-5cb0-4d3a-b021-8aec05314a65, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:24.118875669Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:24.118917023Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 061f99f8-6626-4a81-8262-abd77ceeb36a, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:24.137433857Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:24.137495443Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 061f99f8-6626-4a81-8262-abd77ceeb36a, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 29db7ccb-90e1-414b-aeee-a9393f639631, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:24.155878456Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:24.155929898Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 29db7ccb-90e1-414b-aeee-a9393f639631, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:24.174756644Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:24.174800829Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d857801a-a95b-4937-a2f0-31ad7c239ac7, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:24.193151422Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:24.193197464Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: d857801a-a95b-4937-a2f0-31ad7c239ac7, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:24.211437812Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:24.211490969Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:24.231254132Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:24.231300539Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 58b140e5-f61e-4410-8851-af929dfd8069, dataset_name: DatasetName { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:24.249786995Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:24.249830336Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 58b140e5-f61e-4410-8851-af929dfd8069, dataset_name: DatasetName { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:24.268851441Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:24.268895608Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, dataset_name: DatasetName { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:24.287219358Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:24.287268155Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, dataset_name: DatasetName { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1741444-7ab7-4888-aa3c-e7b6da3527da, dataset_name: DatasetName { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:24.306546296Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:24.306586956Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: c1741444-7ab7-4888-aa3c-e7b6da3527da, dataset_name: DatasetName { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:24.325758416Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Reading ledger from /pool/int/4576fedf-479b-43b3-8fad-bb8b7ec274f8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:24.325809858Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/d09b3f64-0a36-4052-8233-a94e62e662e2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:24.326029021Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:24.326222723Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:24.418829637Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:24.418879382Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:24.418894401Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:24.418899988Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:24.418904301Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:24.418910216Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:24.41891571Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:24.418921438Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:24.418925357Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:24.418932061Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:24.418938483Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:24.418960983Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:24.418966692Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:24.418973367Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:24.418981383Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:24.418988049Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:24.418995315Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:24.419000968Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:24.419006293Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:24.419077319Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"8390633c-d249-468f-8de5-9021d88b894f","remote_addr":"[fd00:1122:3344:101::1]:59148","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf\", TimeNotSynchronized), (\"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069\", TimeNotSynchronized), (\"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451\", TimeNotSynchronized), (\"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da\", TimeNotSynchronized), (\"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb\", TimeNotSynchronized), (\"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d\", TimeNotSynchronized), (\"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8\", TimeNotSynchronized), (\"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\", TimeNotSynchronized), (\"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d\", TimeNotSynchronized), (\"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a\", TimeNotSynchronized), (\"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015\", TimeNotSynchronized), (\"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300\", TimeNotSynchronized), (\"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631\", TimeNotSynchronized), (\"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8\", TimeNotSynchronized), (\"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83\", TimeNotSynchronized), (\"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7\", TimeNotSynchronized), (\"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65\", TimeNotSynchronized), (\"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1\", TimeNotSynchronized), (\"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2\", TimeNotSynchronized)]","latency_us":483851,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:24.419236941Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"5.573290306s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"8390633c-d249-468f-8de5-9021d88b894f\", \"content-length\": \"124\", \"date\": \"Mon, 05 Aug 2024 16:44:23 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"8390633c-d249-468f-8de5-9021d88b894f\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 83a194e2-3464-4a84-ae47-e33a0983bc56, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: fe17df80-010a-4b68-9ebc-6a94437be875, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5b6cee62-004c-4843-9e16-c25d2b0c743b, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6300d678-c957-4259-9a3e-c167ee4d0fb0, kind: Service { id: 5b6cee62-004c-4843-9e16-c25d2b0c743b }, name: Name(\"ntp-5b6cee62-004c-4843-9e16-c25d2b0c743b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 71])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 3c470f26-8422-4139-9f7f-81a0d84becba, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: acae5a41-5da9-420f-9bc8-154e4abdaf87, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } } } }, OmicronZoneConfig { id: eed4ad2f-7765-43e1-8943-ea458ed22e57, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } } } }, OmicronZoneConfig { id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 902ea001-1913-4130-9979-87a8d3db1f53, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8995285b-5cb0-4d3a-b021-8aec05314a65, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: c8555f5b-90c8-4b5b-b73e-5b0ba7ce0695, kind: Service { id: 8995285b-5cb0-4d3a-b021-8aec05314a65 }, name: Name(\"external-dns-8995285b-5cb0-4d3a-b021-8aec05314a65\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 233, 199])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 0a07869d-c9ec-4ade-b114-97ceff54b60e, kind: Service { id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8 }, name: Name(\"external-dns-8a344ff3-4023-4f5e-a29a-3db2dc6980b8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 206, 204])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 866d6328-4ca1-435e-8a98-cd5268df29e8, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 6eedd1d7-7b63-4139-96c3-ad58e9233e7c, kind: Service { id: 866d6328-4ca1-435e-8a98-cd5268df29e8 }, name: Name(\"nexus-866d6328-4ca1-435e-8a98-cd5268df29e8\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 250])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4952ec5d-23ba-48ea-9509-011fcb02adaf, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 38978e97-2109-4173-8624-98de2b729958, kind: Service { id: 4952ec5d-23ba-48ea-9509-011fcb02adaf }, name: Name(\"nexus-4952ec5d-23ba-48ea-9509-011fcb02adaf\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 188])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1c82ddd6-6f5a-4a92-a9b3-fde01c046c42, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 51730dfc-654f-49d5-a0b7-5f1b747fb3be, kind: Service { id: 1c82ddd6-6f5a-4a92-a9b3-fde01c046c42 }, name: Name(\"nexus-1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 28])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 893afdd2-39c6-4f94-b381-facb1a40bbb1, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 061f99f8-6626-4a81-8262-abd77ceeb36a, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 15711896-d3e4-44e3-b330-624f0f5e064d, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: c02f008f-19c7-48c5-93c6-3f776c51e015, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 738351b7-dacb-486a-ab38-cf8aefaef9cb, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 29db7ccb-90e1-414b-aeee-a9393f639631, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } } } }, OmicronZoneConfig { id: d857801a-a95b-4937-a2f0-31ad7c239ac7, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External } } } }, OmicronZoneConfig { id: 58b140e5-f61e-4410-8851-af929dfd8069, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External } } } }, OmicronZoneConfig { id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External } } } }, OmicronZoneConfig { id: c1741444-7ab7-4888-aa3c-e7b6da3527da, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:30.00811144Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83a194e2-3464-4a84-ae47-e33a0983bc56, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:30.008497462Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:30.008573662Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 83a194e2-3464-4a84-ae47-e33a0983bc56, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:30.030781775Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:30.030840848Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe17df80-010a-4b68-9ebc-6a94437be875, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:30.049063914Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:30.049121299Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: fe17df80-010a-4b68-9ebc-6a94437be875, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c470f26-8422-4139-9f7f-81a0d84becba, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:30.069184084Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:30.069236548Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 3c470f26-8422-4139-9f7f-81a0d84becba, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: acae5a41-5da9-420f-9bc8-154e4abdaf87, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:30.090554852Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:30.090599169Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: acae5a41-5da9-420f-9bc8-154e4abdaf87, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eed4ad2f-7765-43e1-8943-ea458ed22e57, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:30.113196335Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:30.113243811Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: eed4ad2f-7765-43e1-8943-ea458ed22e57, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:30.132704319Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:30.134688168Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 902ea001-1913-4130-9979-87a8d3db1f53, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:30.153712619Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:30.153754808Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 902ea001-1913-4130-9979-87a8d3db1f53, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8995285b-5cb0-4d3a-b021-8aec05314a65, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:30.171536529Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:30.171589302Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8995285b-5cb0-4d3a-b021-8aec05314a65, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:30.189633451Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:30.189680045Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 061f99f8-6626-4a81-8262-abd77ceeb36a, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:30.206888881Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:30.206937903Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 061f99f8-6626-4a81-8262-abd77ceeb36a, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 29db7ccb-90e1-414b-aeee-a9393f639631, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:30.225532921Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:30.225572184Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 29db7ccb-90e1-414b-aeee-a9393f639631, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:30.245592187Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:30.245620885Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d857801a-a95b-4937-a2f0-31ad7c239ac7, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:30.263742246Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:30.26377341Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: d857801a-a95b-4937-a2f0-31ad7c239ac7, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:30.282243071Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:30.282337772Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:30.300995887Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:30.301031729Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 58b140e5-f61e-4410-8851-af929dfd8069, dataset_name: DatasetName { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:30.319051359Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:30.319102632Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 58b140e5-f61e-4410-8851-af929dfd8069, dataset_name: DatasetName { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:30.338350302Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:30.338401809Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, dataset_name: DatasetName { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:30.357065411Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:30.357102855Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, dataset_name: DatasetName { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1741444-7ab7-4888-aa3c-e7b6da3527da, dataset_name: DatasetName { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:30.375903001Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:30.375939442Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: c1741444-7ab7-4888-aa3c-e7b6da3527da, dataset_name: DatasetName { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:30.394307882Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Reading ledger from /pool/int/4576fedf-479b-43b3-8fad-bb8b7ec274f8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:30.394352799Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/d09b3f64-0a36-4052-8233-a94e62e662e2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:30.394545253Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:30.394735031Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:30.4875437Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:30.487597729Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:30.487609954Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:30.487617379Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:30.487623361Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:30.487633946Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:30.487639515Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:30.487646433Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:30.487651655Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:30.487657102Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:30.487662324Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:30.487669018Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:30.487680137Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:30.487685377Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:30.487690712Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:30.487708375Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:30.487714056Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:30.487720928Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:30.487731616Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:30.487803064Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"4e409dc8-4724-4109-bdc2-ef1f9df367c4","remote_addr":"[fd00:1122:3344:101::1]:59148","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451\", TimeNotSynchronized), (\"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1\", TimeNotSynchronized), (\"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300\", TimeNotSynchronized), (\"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\", TimeNotSynchronized), (\"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da\", TimeNotSynchronized), (\"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8\", TimeNotSynchronized), (\"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d\", TimeNotSynchronized), (\"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65\", TimeNotSynchronized), (\"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb\", TimeNotSynchronized), (\"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7\", TimeNotSynchronized), (\"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2\", TimeNotSynchronized), (\"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8\", TimeNotSynchronized), (\"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf\", TimeNotSynchronized), (\"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d\", TimeNotSynchronized), (\"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069\", TimeNotSynchronized), (\"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a\", TimeNotSynchronized), (\"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83\", TimeNotSynchronized), (\"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015\", TimeNotSynchronized), (\"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631\", TimeNotSynchronized)]","latency_us":478188,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:30.488028096Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"4.64298186s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"4e409dc8-4724-4109-bdc2-ef1f9df367c4\", \"content-length\": \"124\", \"date\": \"Mon, 05 Aug 2024 16:44:30 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"4e409dc8-4724-4109-bdc2-ef1f9df367c4\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 83a194e2-3464-4a84-ae47-e33a0983bc56, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: fe17df80-010a-4b68-9ebc-6a94437be875, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5b6cee62-004c-4843-9e16-c25d2b0c743b, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6300d678-c957-4259-9a3e-c167ee4d0fb0, kind: Service { id: 5b6cee62-004c-4843-9e16-c25d2b0c743b }, name: Name(\"ntp-5b6cee62-004c-4843-9e16-c25d2b0c743b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 71])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 3c470f26-8422-4139-9f7f-81a0d84becba, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: acae5a41-5da9-420f-9bc8-154e4abdaf87, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } } } }, OmicronZoneConfig { id: eed4ad2f-7765-43e1-8943-ea458ed22e57, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } } } }, OmicronZoneConfig { id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 902ea001-1913-4130-9979-87a8d3db1f53, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8995285b-5cb0-4d3a-b021-8aec05314a65, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: c8555f5b-90c8-4b5b-b73e-5b0ba7ce0695, kind: Service { id: 8995285b-5cb0-4d3a-b021-8aec05314a65 }, name: Name(\"external-dns-8995285b-5cb0-4d3a-b021-8aec05314a65\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 233, 199])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 0a07869d-c9ec-4ade-b114-97ceff54b60e, kind: Service { id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8 }, name: Name(\"external-dns-8a344ff3-4023-4f5e-a29a-3db2dc6980b8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 206, 204])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 866d6328-4ca1-435e-8a98-cd5268df29e8, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 6eedd1d7-7b63-4139-96c3-ad58e9233e7c, kind: Service { id: 866d6328-4ca1-435e-8a98-cd5268df29e8 }, name: Name(\"nexus-866d6328-4ca1-435e-8a98-cd5268df29e8\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 250])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4952ec5d-23ba-48ea-9509-011fcb02adaf, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 38978e97-2109-4173-8624-98de2b729958, kind: Service { id: 4952ec5d-23ba-48ea-9509-011fcb02adaf }, name: Name(\"nexus-4952ec5d-23ba-48ea-9509-011fcb02adaf\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 188])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1c82ddd6-6f5a-4a92-a9b3-fde01c046c42, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 51730dfc-654f-49d5-a0b7-5f1b747fb3be, kind: Service { id: 1c82ddd6-6f5a-4a92-a9b3-fde01c046c42 }, name: Name(\"nexus-1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 28])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 893afdd2-39c6-4f94-b381-facb1a40bbb1, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 061f99f8-6626-4a81-8262-abd77ceeb36a, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 15711896-d3e4-44e3-b330-624f0f5e064d, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: c02f008f-19c7-48c5-93c6-3f776c51e015, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 738351b7-dacb-486a-ab38-cf8aefaef9cb, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 29db7ccb-90e1-414b-aeee-a9393f639631, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } } } }, OmicronZoneConfig { id: d857801a-a95b-4937-a2f0-31ad7c239ac7, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External } } } }, OmicronZoneConfig { id: 58b140e5-f61e-4410-8851-af929dfd8069, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External } } } }, OmicronZoneConfig { id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External } } } }, OmicronZoneConfig { id: c1741444-7ab7-4888-aa3c-e7b6da3527da, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:35.144059383Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83a194e2-3464-4a84-ae47-e33a0983bc56, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:35.144953027Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:35.144961033Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 83a194e2-3464-4a84-ae47-e33a0983bc56, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:35.168026683Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:35.168074403Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe17df80-010a-4b68-9ebc-6a94437be875, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:35.188175475Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:35.188217279Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: fe17df80-010a-4b68-9ebc-6a94437be875, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c470f26-8422-4139-9f7f-81a0d84becba, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:35.207589261Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:35.207702629Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 3c470f26-8422-4139-9f7f-81a0d84becba, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: acae5a41-5da9-420f-9bc8-154e4abdaf87, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:35.226973558Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:35.227018203Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: acae5a41-5da9-420f-9bc8-154e4abdaf87, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eed4ad2f-7765-43e1-8943-ea458ed22e57, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:35.244581571Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:35.244627996Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: eed4ad2f-7765-43e1-8943-ea458ed22e57, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:35.26227055Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:35.262321054Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 902ea001-1913-4130-9979-87a8d3db1f53, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:35.28047557Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:35.280512312Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 902ea001-1913-4130-9979-87a8d3db1f53, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8995285b-5cb0-4d3a-b021-8aec05314a65, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:35.298602364Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:35.298638927Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8995285b-5cb0-4d3a-b021-8aec05314a65, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:35.316985671Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:35.317023462Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 061f99f8-6626-4a81-8262-abd77ceeb36a, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:35.335071112Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:35.335118776Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 061f99f8-6626-4a81-8262-abd77ceeb36a, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 29db7ccb-90e1-414b-aeee-a9393f639631, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:35.353561263Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:35.353598258Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 29db7ccb-90e1-414b-aeee-a9393f639631, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:35.371804181Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:35.371838082Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d857801a-a95b-4937-a2f0-31ad7c239ac7, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:35.390254202Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:35.390299025Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: d857801a-a95b-4937-a2f0-31ad7c239ac7, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:35.408831332Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:35.408870951Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:35.426749128Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:35.4267854Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 58b140e5-f61e-4410-8851-af929dfd8069, dataset_name: DatasetName { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:35.444978765Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:35.445019922Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 58b140e5-f61e-4410-8851-af929dfd8069, dataset_name: DatasetName { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:35.465059842Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:35.465107187Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, dataset_name: DatasetName { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:35.483146079Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:35.483194493Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, dataset_name: DatasetName { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1741444-7ab7-4888-aa3c-e7b6da3527da, dataset_name: DatasetName { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:35.501656361Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:35.501702749Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: c1741444-7ab7-4888-aa3c-e7b6da3527da, dataset_name: DatasetName { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:35.520064281Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Reading ledger from /pool/int/4576fedf-479b-43b3-8fad-bb8b7ec274f8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:35.520122782Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/d09b3f64-0a36-4052-8233-a94e62e662e2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:35.520356626Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:35.520546306Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:35.613957649Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:35.61401405Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:35.614028647Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:35.614053445Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:35.614059239Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:35.614066636Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:35.614074201Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:35.614078776Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:35.614084608Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:35.614088508Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:35.614093861Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:35.614099458Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:35.614104418Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:35.614109649Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:35.614120177Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:35.614128708Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:35.61413274Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:35.614137887Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:35.614141787Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:35.614162131Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"145103fd-4ba8-4039-a47d-33d5b5b2bfff","remote_addr":"[fd00:1122:3344:101::1]:59148","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65\", TimeNotSynchronized), (\"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631\", TimeNotSynchronized), (\"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300\", TimeNotSynchronized), (\"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\", TimeNotSynchronized), (\"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1\", TimeNotSynchronized), (\"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8\", TimeNotSynchronized), (\"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d\", TimeNotSynchronized), (\"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069\", TimeNotSynchronized), (\"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf\", TimeNotSynchronized), (\"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2\", TimeNotSynchronized), (\"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7\", TimeNotSynchronized), (\"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451\", TimeNotSynchronized), (\"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8\", TimeNotSynchronized), (\"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb\", TimeNotSynchronized), (\"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015\", TimeNotSynchronized), (\"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d\", TimeNotSynchronized), (\"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83\", TimeNotSynchronized), (\"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a\", TimeNotSynchronized), (\"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da\", TimeNotSynchronized)]","latency_us":468664,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:35.614346793Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"9.864373544s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"145103fd-4ba8-4039-a47d-33d5b5b2bfff\", \"content-length\": \"124\", \"date\": \"Mon, 05 Aug 2024 16:44:35 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"145103fd-4ba8-4039-a47d-33d5b5b2bfff\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:40.644962431Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","component":"MetricsManager","sled_id":"3abf3a4f-8820-40af-a01f-4369b42a5529","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"10.16110074s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 83a194e2-3464-4a84-ae47-e33a0983bc56, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: fe17df80-010a-4b68-9ebc-6a94437be875, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5b6cee62-004c-4843-9e16-c25d2b0c743b, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6300d678-c957-4259-9a3e-c167ee4d0fb0, kind: Service { id: 5b6cee62-004c-4843-9e16-c25d2b0c743b }, name: Name(\"ntp-5b6cee62-004c-4843-9e16-c25d2b0c743b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 71])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 3c470f26-8422-4139-9f7f-81a0d84becba, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: acae5a41-5da9-420f-9bc8-154e4abdaf87, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } } } }, OmicronZoneConfig { id: eed4ad2f-7765-43e1-8943-ea458ed22e57, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } } } }, OmicronZoneConfig { id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 902ea001-1913-4130-9979-87a8d3db1f53, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8995285b-5cb0-4d3a-b021-8aec05314a65, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: c8555f5b-90c8-4b5b-b73e-5b0ba7ce0695, kind: Service { id: 8995285b-5cb0-4d3a-b021-8aec05314a65 }, name: Name(\"external-dns-8995285b-5cb0-4d3a-b021-8aec05314a65\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 233, 199])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 0a07869d-c9ec-4ade-b114-97ceff54b60e, kind: Service { id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8 }, name: Name(\"external-dns-8a344ff3-4023-4f5e-a29a-3db2dc6980b8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 206, 204])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 866d6328-4ca1-435e-8a98-cd5268df29e8, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 6eedd1d7-7b63-4139-96c3-ad58e9233e7c, kind: Service { id: 866d6328-4ca1-435e-8a98-cd5268df29e8 }, name: Name(\"nexus-866d6328-4ca1-435e-8a98-cd5268df29e8\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 250])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4952ec5d-23ba-48ea-9509-011fcb02adaf, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 38978e97-2109-4173-8624-98de2b729958, kind: Service { id: 4952ec5d-23ba-48ea-9509-011fcb02adaf }, name: Name(\"nexus-4952ec5d-23ba-48ea-9509-011fcb02adaf\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 188])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1c82ddd6-6f5a-4a92-a9b3-fde01c046c42, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 51730dfc-654f-49d5-a0b7-5f1b747fb3be, kind: Service { id: 1c82ddd6-6f5a-4a92-a9b3-fde01c046c42 }, name: Name(\"nexus-1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 28])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 893afdd2-39c6-4f94-b381-facb1a40bbb1, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 061f99f8-6626-4a81-8262-abd77ceeb36a, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 15711896-d3e4-44e3-b330-624f0f5e064d, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: c02f008f-19c7-48c5-93c6-3f776c51e015, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 738351b7-dacb-486a-ab38-cf8aefaef9cb, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 29db7ccb-90e1-414b-aeee-a9393f639631, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } } } }, OmicronZoneConfig { id: d857801a-a95b-4937-a2f0-31ad7c239ac7, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External } } } }, OmicronZoneConfig { id: 58b140e5-f61e-4410-8851-af929dfd8069, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External } } } }, OmicronZoneConfig { id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External } } } }, OmicronZoneConfig { id: c1741444-7ab7-4888-aa3c-e7b6da3527da, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:45.506242195Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83a194e2-3464-4a84-ae47-e33a0983bc56, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:45.506490056Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:45.506499768Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 83a194e2-3464-4a84-ae47-e33a0983bc56, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:45.528514679Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:45.528574361Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe17df80-010a-4b68-9ebc-6a94437be875, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:45.548189199Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:45.54823211Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: fe17df80-010a-4b68-9ebc-6a94437be875, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c470f26-8422-4139-9f7f-81a0d84becba, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:45.565849902Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:45.565902703Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 3c470f26-8422-4139-9f7f-81a0d84becba, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: acae5a41-5da9-420f-9bc8-154e4abdaf87, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:45.583606026Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:45.58364918Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: acae5a41-5da9-420f-9bc8-154e4abdaf87, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eed4ad2f-7765-43e1-8943-ea458ed22e57, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:45.601991017Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:45.60203493Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: eed4ad2f-7765-43e1-8943-ea458ed22e57, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:45.619003629Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:45.619057517Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 902ea001-1913-4130-9979-87a8d3db1f53, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:45.639039384Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:45.641011595Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 902ea001-1913-4130-9979-87a8d3db1f53, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8995285b-5cb0-4d3a-b021-8aec05314a65, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:45.657676763Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:45.657742586Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8995285b-5cb0-4d3a-b021-8aec05314a65, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:45.679132184Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:45.679184301Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 061f99f8-6626-4a81-8262-abd77ceeb36a, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:45.69832272Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:45.698367074Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 061f99f8-6626-4a81-8262-abd77ceeb36a, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 29db7ccb-90e1-414b-aeee-a9393f639631, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:45.717474602Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:45.717512346Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 29db7ccb-90e1-414b-aeee-a9393f639631, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:45.735942814Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:45.735999759Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d857801a-a95b-4937-a2f0-31ad7c239ac7, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:45.75514061Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:45.755189277Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: d857801a-a95b-4937-a2f0-31ad7c239ac7, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:45.774171036Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:45.774219665Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:45.790742135Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:45.790798742Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 58b140e5-f61e-4410-8851-af929dfd8069, dataset_name: DatasetName { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:45.808695196Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:45.808740974Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 58b140e5-f61e-4410-8851-af929dfd8069, dataset_name: DatasetName { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:45.826996288Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:45.827042977Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, dataset_name: DatasetName { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:45.845835219Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:45.845884889Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, dataset_name: DatasetName { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1741444-7ab7-4888-aa3c-e7b6da3527da, dataset_name: DatasetName { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:45.864960479Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:45.865005339Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: c1741444-7ab7-4888-aa3c-e7b6da3527da, dataset_name: DatasetName { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:45.882369858Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Reading ledger from /pool/int/4576fedf-479b-43b3-8fad-bb8b7ec274f8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:45.882424103Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/d09b3f64-0a36-4052-8233-a94e62e662e2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:45.882839451Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:45.882975222Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:45.976933152Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:45.976983545Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:45.976992113Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:45.976998835Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:45.977006016Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:45.977030645Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:45.977039035Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:45.977047183Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:45.977054917Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:45.977064517Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:45.977072505Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:45.977078514Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:45.97708503Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:45.977089586Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:45.97709598Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:45.977104437Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:45.977112265Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:45.97712554Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:45.97713065Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:45.977173269Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"5e22d560-bc0c-4550-beea-c668930c2d24","remote_addr":"[fd00:1122:3344:101::1]:59148","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65\", TimeNotSynchronized), (\"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300\", TimeNotSynchronized), (\"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015\", TimeNotSynchronized), (\"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb\", TimeNotSynchronized), (\"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da\", TimeNotSynchronized), (\"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\", TimeNotSynchronized), (\"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d\", TimeNotSynchronized), (\"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf\", TimeNotSynchronized), (\"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d\", TimeNotSynchronized), (\"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1\", TimeNotSynchronized), (\"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2\", TimeNotSynchronized), (\"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069\", TimeNotSynchronized), (\"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8\", TimeNotSynchronized), (\"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a\", TimeNotSynchronized), (\"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631\", TimeNotSynchronized), (\"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8\", TimeNotSynchronized), (\"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83\", TimeNotSynchronized), (\"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7\", TimeNotSynchronized), (\"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451\", TimeNotSynchronized)]","latency_us":469553,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:45.977332122Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"4.900025189s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"5e22d560-bc0c-4550-beea-c668930c2d24\", \"content-length\": \"124\", \"date\": \"Mon, 05 Aug 2024 16:44:45 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"5e22d560-bc0c-4550-beea-c668930c2d24\" }"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 83a194e2-3464-4a84-ae47-e33a0983bc56, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: fe17df80-010a-4b68-9ebc-6a94437be875, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5b6cee62-004c-4843-9e16-c25d2b0c743b, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6300d678-c957-4259-9a3e-c167ee4d0fb0, kind: Service { id: 5b6cee62-004c-4843-9e16-c25d2b0c743b }, name: Name(\"ntp-5b6cee62-004c-4843-9e16-c25d2b0c743b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 71])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 3c470f26-8422-4139-9f7f-81a0d84becba, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: acae5a41-5da9-420f-9bc8-154e4abdaf87, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } } } }, OmicronZoneConfig { id: eed4ad2f-7765-43e1-8943-ea458ed22e57, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } } } }, OmicronZoneConfig { id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 902ea001-1913-4130-9979-87a8d3db1f53, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8995285b-5cb0-4d3a-b021-8aec05314a65, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: c8555f5b-90c8-4b5b-b73e-5b0ba7ce0695, kind: Service { id: 8995285b-5cb0-4d3a-b021-8aec05314a65 }, name: Name(\"external-dns-8995285b-5cb0-4d3a-b021-8aec05314a65\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 233, 199])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 0a07869d-c9ec-4ade-b114-97ceff54b60e, kind: Service { id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8 }, name: Name(\"external-dns-8a344ff3-4023-4f5e-a29a-3db2dc6980b8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 206, 204])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 866d6328-4ca1-435e-8a98-cd5268df29e8, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 6eedd1d7-7b63-4139-96c3-ad58e9233e7c, kind: Service { id: 866d6328-4ca1-435e-8a98-cd5268df29e8 }, name: Name(\"nexus-866d6328-4ca1-435e-8a98-cd5268df29e8\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 250])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4952ec5d-23ba-48ea-9509-011fcb02adaf, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 38978e97-2109-4173-8624-98de2b729958, kind: Service { id: 4952ec5d-23ba-48ea-9509-011fcb02adaf }, name: Name(\"nexus-4952ec5d-23ba-48ea-9509-011fcb02adaf\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 188])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1c82ddd6-6f5a-4a92-a9b3-fde01c046c42, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 51730dfc-654f-49d5-a0b7-5f1b747fb3be, kind: Service { id: 1c82ddd6-6f5a-4a92-a9b3-fde01c046c42 }, name: Name(\"nexus-1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 28])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 893afdd2-39c6-4f94-b381-facb1a40bbb1, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 061f99f8-6626-4a81-8262-abd77ceeb36a, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 15711896-d3e4-44e3-b330-624f0f5e064d, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: c02f008f-19c7-48c5-93c6-3f776c51e015, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 738351b7-dacb-486a-ab38-cf8aefaef9cb, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 29db7ccb-90e1-414b-aeee-a9393f639631, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } } } }, OmicronZoneConfig { id: d857801a-a95b-4937-a2f0-31ad7c239ac7, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External } } } }, OmicronZoneConfig { id: 58b140e5-f61e-4410-8851-af929dfd8069, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External } } } }, OmicronZoneConfig { id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External } } } }, OmicronZoneConfig { id: c1741444-7ab7-4888-aa3c-e7b6da3527da, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:50.892244527Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83a194e2-3464-4a84-ae47-e33a0983bc56, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:50.89255567Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:50.892575629Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 83a194e2-3464-4a84-ae47-e33a0983bc56, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:50.914560613Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:50.914626079Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe17df80-010a-4b68-9ebc-6a94437be875, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:50.934609841Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:50.934656745Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: fe17df80-010a-4b68-9ebc-6a94437be875, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c470f26-8422-4139-9f7f-81a0d84becba, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:50.952467374Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:50.952506365Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 3c470f26-8422-4139-9f7f-81a0d84becba, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: acae5a41-5da9-420f-9bc8-154e4abdaf87, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:50.970570757Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:50.970618092Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: acae5a41-5da9-420f-9bc8-154e4abdaf87, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eed4ad2f-7765-43e1-8943-ea458ed22e57, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:50.990422124Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:50.990542014Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: eed4ad2f-7765-43e1-8943-ea458ed22e57, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:51.008356872Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:51.008403083Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 902ea001-1913-4130-9979-87a8d3db1f53, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:51.026364628Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:51.026422651Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 902ea001-1913-4130-9979-87a8d3db1f53, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8995285b-5cb0-4d3a-b021-8aec05314a65, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:51.0453756Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:51.045424979Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8995285b-5cb0-4d3a-b021-8aec05314a65, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:51.063395107Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:51.063437333Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 061f99f8-6626-4a81-8262-abd77ceeb36a, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:51.080549918Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:51.080595763Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 061f99f8-6626-4a81-8262-abd77ceeb36a, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 29db7ccb-90e1-414b-aeee-a9393f639631, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:51.09940208Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:51.099445122Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 29db7ccb-90e1-414b-aeee-a9393f639631, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:51.117976808Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:51.118027087Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d857801a-a95b-4937-a2f0-31ad7c239ac7, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:51.137798056Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:51.137915477Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: d857801a-a95b-4937-a2f0-31ad7c239ac7, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:51.156405977Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:51.156455225Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:51.175553528Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:51.175606057Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 58b140e5-f61e-4410-8851-af929dfd8069, dataset_name: DatasetName { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:51.194375462Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:51.19668509Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 58b140e5-f61e-4410-8851-af929dfd8069, dataset_name: DatasetName { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:51.213483596Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:51.213524284Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, dataset_name: DatasetName { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:51.231923135Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:51.231968483Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, dataset_name: DatasetName { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1741444-7ab7-4888-aa3c-e7b6da3527da, dataset_name: DatasetName { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:51.25125891Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:51.251312808Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: c1741444-7ab7-4888-aa3c-e7b6da3527da, dataset_name: DatasetName { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:51.270894975Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Reading ledger from /pool/int/4576fedf-479b-43b3-8fad-bb8b7ec274f8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:51.27094847Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/d09b3f64-0a36-4052-8233-a94e62e662e2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:51.271113313Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:51.271329575Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:51.364727549Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:51.364793625Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:51.364816914Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:51.364825736Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:51.364833695Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:51.364839076Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:51.364846201Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:51.36485627Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:51.364862955Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:51.364889196Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:51.364909268Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:51.364916056Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:51.364921109Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:51.364927831Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:51.364934347Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:51.364942559Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:51.364949084Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:51.364955309Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:51.3649598Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:51.365003254Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"f69fccc9-7448-4256-8a45-c473dba8d6c7","remote_addr":"[fd00:1122:3344:101::1]:59148","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb\", TimeNotSynchronized), (\"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65\", TimeNotSynchronized), (\"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015\", TimeNotSynchronized), (\"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069\", TimeNotSynchronized), (\"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451\", TimeNotSynchronized), (\"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300\", TimeNotSynchronized), (\"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83\", TimeNotSynchronized), (\"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d\", TimeNotSynchronized), (\"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a\", TimeNotSynchronized), (\"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da\", TimeNotSynchronized), (\"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2\", TimeNotSynchronized), (\"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf\", TimeNotSynchronized), (\"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8\", TimeNotSynchronized), (\"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7\", TimeNotSynchronized), (\"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d\", TimeNotSynchronized), (\"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8\", TimeNotSynchronized), (\"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1\", TimeNotSynchronized), (\"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631\", TimeNotSynchronized), (\"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\", TimeNotSynchronized)]","latency_us":471398,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:44:51.365179629Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"11.122751636s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"f69fccc9-7448-4256-8a45-c473dba8d6c7\", \"content-length\": \"124\", \"date\": \"Mon, 05 Aug 2024 16:44:51 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"f69fccc9-7448-4256-8a45-c473dba8d6c7\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:53.541330233Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:59039"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:44:53.542106174Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"b1436ad7-9e32-44a2-8c09-dce965842009","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:59039","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":709,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 83a194e2-3464-4a84-ae47-e33a0983bc56, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: fe17df80-010a-4b68-9ebc-6a94437be875, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5b6cee62-004c-4843-9e16-c25d2b0c743b, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6300d678-c957-4259-9a3e-c167ee4d0fb0, kind: Service { id: 5b6cee62-004c-4843-9e16-c25d2b0c743b }, name: Name(\"ntp-5b6cee62-004c-4843-9e16-c25d2b0c743b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 71])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 3c470f26-8422-4139-9f7f-81a0d84becba, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: acae5a41-5da9-420f-9bc8-154e4abdaf87, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } } } }, OmicronZoneConfig { id: eed4ad2f-7765-43e1-8943-ea458ed22e57, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } } } }, OmicronZoneConfig { id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 902ea001-1913-4130-9979-87a8d3db1f53, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8995285b-5cb0-4d3a-b021-8aec05314a65, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: c8555f5b-90c8-4b5b-b73e-5b0ba7ce0695, kind: Service { id: 8995285b-5cb0-4d3a-b021-8aec05314a65 }, name: Name(\"external-dns-8995285b-5cb0-4d3a-b021-8aec05314a65\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 233, 199])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 0a07869d-c9ec-4ade-b114-97ceff54b60e, kind: Service { id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8 }, name: Name(\"external-dns-8a344ff3-4023-4f5e-a29a-3db2dc6980b8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 206, 204])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 866d6328-4ca1-435e-8a98-cd5268df29e8, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 6eedd1d7-7b63-4139-96c3-ad58e9233e7c, kind: Service { id: 866d6328-4ca1-435e-8a98-cd5268df29e8 }, name: Name(\"nexus-866d6328-4ca1-435e-8a98-cd5268df29e8\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 250])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4952ec5d-23ba-48ea-9509-011fcb02adaf, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 38978e97-2109-4173-8624-98de2b729958, kind: Service { id: 4952ec5d-23ba-48ea-9509-011fcb02adaf }, name: Name(\"nexus-4952ec5d-23ba-48ea-9509-011fcb02adaf\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 188])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1c82ddd6-6f5a-4a92-a9b3-fde01c046c42, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 51730dfc-654f-49d5-a0b7-5f1b747fb3be, kind: Service { id: 1c82ddd6-6f5a-4a92-a9b3-fde01c046c42 }, name: Name(\"nexus-1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 28])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 893afdd2-39c6-4f94-b381-facb1a40bbb1, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 061f99f8-6626-4a81-8262-abd77ceeb36a, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 15711896-d3e4-44e3-b330-624f0f5e064d, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: c02f008f-19c7-48c5-93c6-3f776c51e015, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 738351b7-dacb-486a-ab38-cf8aefaef9cb, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 29db7ccb-90e1-414b-aeee-a9393f639631, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } } } }, OmicronZoneConfig { id: d857801a-a95b-4937-a2f0-31ad7c239ac7, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External } } } }, OmicronZoneConfig { id: 58b140e5-f61e-4410-8851-af929dfd8069, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External } } } }, OmicronZoneConfig { id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External } } } }, OmicronZoneConfig { id: c1741444-7ab7-4888-aa3c-e7b6da3527da, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:02.519977314Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83a194e2-3464-4a84-ae47-e33a0983bc56, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:02.521114782Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:02.521125114Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 83a194e2-3464-4a84-ae47-e33a0983bc56, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:02.545021488Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:02.545075123Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe17df80-010a-4b68-9ebc-6a94437be875, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:02.566186789Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:02.566237349Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: fe17df80-010a-4b68-9ebc-6a94437be875, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c470f26-8422-4139-9f7f-81a0d84becba, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:02.588293138Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:02.588342733Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 3c470f26-8422-4139-9f7f-81a0d84becba, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: acae5a41-5da9-420f-9bc8-154e4abdaf87, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:02.607581277Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:02.607622068Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: acae5a41-5da9-420f-9bc8-154e4abdaf87, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eed4ad2f-7765-43e1-8943-ea458ed22e57, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:02.627564651Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:02.627609811Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: eed4ad2f-7765-43e1-8943-ea458ed22e57, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:02.645424722Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:02.645471709Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 902ea001-1913-4130-9979-87a8d3db1f53, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:02.663243128Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:02.66328826Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 902ea001-1913-4130-9979-87a8d3db1f53, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8995285b-5cb0-4d3a-b021-8aec05314a65, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:02.680979916Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:02.681032783Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8995285b-5cb0-4d3a-b021-8aec05314a65, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:02.700122231Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:02.700172884Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 061f99f8-6626-4a81-8262-abd77ceeb36a, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:02.719483686Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:02.719526493Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 061f99f8-6626-4a81-8262-abd77ceeb36a, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 29db7ccb-90e1-414b-aeee-a9393f639631, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:02.739003499Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:02.739048386Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 29db7ccb-90e1-414b-aeee-a9393f639631, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:02.757546635Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:02.757587051Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d857801a-a95b-4937-a2f0-31ad7c239ac7, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:02.775490165Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:02.775528819Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: d857801a-a95b-4937-a2f0-31ad7c239ac7, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:02.795174311Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:02.795216724Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:02.813930349Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:02.813982587Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 58b140e5-f61e-4410-8851-af929dfd8069, dataset_name: DatasetName { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:02.832518772Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:02.832627669Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 58b140e5-f61e-4410-8851-af929dfd8069, dataset_name: DatasetName { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:02.852332835Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:02.852377461Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, dataset_name: DatasetName { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:02.871137038Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:02.871176986Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, dataset_name: DatasetName { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1741444-7ab7-4888-aa3c-e7b6da3527da, dataset_name: DatasetName { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:02.890286037Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:02.890341191Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: c1741444-7ab7-4888-aa3c-e7b6da3527da, dataset_name: DatasetName { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:02.90856212Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Reading ledger from /pool/int/4576fedf-479b-43b3-8fad-bb8b7ec274f8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:02.908611011Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/d09b3f64-0a36-4052-8233-a94e62e662e2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:02.908914102Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:02.909036457Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:45:03.005085737Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:45:03.005146835Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:45:03.005156575Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:45:03.005171107Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:45:03.005179095Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:45:03.005187382Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:45:03.005193823Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:45:03.005206751Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:45:03.005213839Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:45:03.005230199Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:45:03.005243408Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:45:03.005251461Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:45:03.005258127Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:45:03.005277Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:45:03.005285831Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:45:03.005292375Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:45:03.005303981Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:45:03.005309344Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:45:03.005316019Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:03.005385424Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"ec9da5dc-1abc-43f1-b868-784869382139","remote_addr":"[fd00:1122:3344:101::1]:59148","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\", TimeNotSynchronized), (\"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1\", TimeNotSynchronized), (\"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d\", TimeNotSynchronized), (\"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb\", TimeNotSynchronized), (\"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631\", TimeNotSynchronized), (\"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83\", TimeNotSynchronized), (\"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7\", TimeNotSynchronized), (\"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8\", TimeNotSynchronized), (\"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069\", TimeNotSynchronized), (\"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d\", TimeNotSynchronized), (\"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451\", TimeNotSynchronized), (\"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300\", TimeNotSynchronized), (\"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015\", TimeNotSynchronized), (\"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65\", TimeNotSynchronized), (\"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da\", TimeNotSynchronized), (\"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2\", TimeNotSynchronized), (\"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf\", TimeNotSynchronized), (\"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a\", TimeNotSynchronized), (\"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8\", TimeNotSynchronized)]","latency_us":483911,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:45:03.005576723Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"15.492124441s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"ec9da5dc-1abc-43f1-b868-784869382139\", \"content-length\": \"124\", \"date\": \"Mon, 05 Aug 2024 16:45:02 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"ec9da5dc-1abc-43f1-b868-784869382139\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:45:05.912682646Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","component":"MetricsManager","sled_id":"3abf3a4f-8820-40af-a01f-4369b42a5529","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"15.896631552s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 83a194e2-3464-4a84-ae47-e33a0983bc56, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: fe17df80-010a-4b68-9ebc-6a94437be875, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5b6cee62-004c-4843-9e16-c25d2b0c743b, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6300d678-c957-4259-9a3e-c167ee4d0fb0, kind: Service { id: 5b6cee62-004c-4843-9e16-c25d2b0c743b }, name: Name(\"ntp-5b6cee62-004c-4843-9e16-c25d2b0c743b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 71])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 3c470f26-8422-4139-9f7f-81a0d84becba, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: acae5a41-5da9-420f-9bc8-154e4abdaf87, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } } } }, OmicronZoneConfig { id: eed4ad2f-7765-43e1-8943-ea458ed22e57, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } } } }, OmicronZoneConfig { id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 902ea001-1913-4130-9979-87a8d3db1f53, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8995285b-5cb0-4d3a-b021-8aec05314a65, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: c8555f5b-90c8-4b5b-b73e-5b0ba7ce0695, kind: Service { id: 8995285b-5cb0-4d3a-b021-8aec05314a65 }, name: Name(\"external-dns-8995285b-5cb0-4d3a-b021-8aec05314a65\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 233, 199])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 0a07869d-c9ec-4ade-b114-97ceff54b60e, kind: Service { id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8 }, name: Name(\"external-dns-8a344ff3-4023-4f5e-a29a-3db2dc6980b8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 206, 204])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 866d6328-4ca1-435e-8a98-cd5268df29e8, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 6eedd1d7-7b63-4139-96c3-ad58e9233e7c, kind: Service { id: 866d6328-4ca1-435e-8a98-cd5268df29e8 }, name: Name(\"nexus-866d6328-4ca1-435e-8a98-cd5268df29e8\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 250])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4952ec5d-23ba-48ea-9509-011fcb02adaf, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 38978e97-2109-4173-8624-98de2b729958, kind: Service { id: 4952ec5d-23ba-48ea-9509-011fcb02adaf }, name: Name(\"nexus-4952ec5d-23ba-48ea-9509-011fcb02adaf\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 188])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1c82ddd6-6f5a-4a92-a9b3-fde01c046c42, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 51730dfc-654f-49d5-a0b7-5f1b747fb3be, kind: Service { id: 1c82ddd6-6f5a-4a92-a9b3-fde01c046c42 }, name: Name(\"nexus-1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 28])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 893afdd2-39c6-4f94-b381-facb1a40bbb1, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 061f99f8-6626-4a81-8262-abd77ceeb36a, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 15711896-d3e4-44e3-b330-624f0f5e064d, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: c02f008f-19c7-48c5-93c6-3f776c51e015, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 738351b7-dacb-486a-ab38-cf8aefaef9cb, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 29db7ccb-90e1-414b-aeee-a9393f639631, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } } } }, OmicronZoneConfig { id: d857801a-a95b-4937-a2f0-31ad7c239ac7, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External } } } }, OmicronZoneConfig { id: 58b140e5-f61e-4410-8851-af929dfd8069, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External } } } }, OmicronZoneConfig { id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External } } } }, OmicronZoneConfig { id: c1741444-7ab7-4888-aa3c-e7b6da3527da, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:18.54162395Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83a194e2-3464-4a84-ae47-e33a0983bc56, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:18.542128539Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:18.542143269Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 83a194e2-3464-4a84-ae47-e33a0983bc56, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:18.563386218Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:18.565748466Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe17df80-010a-4b68-9ebc-6a94437be875, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:18.583909163Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:18.583961598Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: fe17df80-010a-4b68-9ebc-6a94437be875, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c470f26-8422-4139-9f7f-81a0d84becba, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:18.604045266Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:18.604096773Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 3c470f26-8422-4139-9f7f-81a0d84becba, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: acae5a41-5da9-420f-9bc8-154e4abdaf87, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:18.622527257Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:18.622575905Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: acae5a41-5da9-420f-9bc8-154e4abdaf87, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eed4ad2f-7765-43e1-8943-ea458ed22e57, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:18.641712964Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:18.643692935Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: eed4ad2f-7765-43e1-8943-ea458ed22e57, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:18.660776672Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:18.660824156Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 902ea001-1913-4130-9979-87a8d3db1f53, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:18.679457652Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:18.679512337Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 902ea001-1913-4130-9979-87a8d3db1f53, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8995285b-5cb0-4d3a-b021-8aec05314a65, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:18.698130477Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:18.69817544Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8995285b-5cb0-4d3a-b021-8aec05314a65, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:18.717052919Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:18.717095669Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 061f99f8-6626-4a81-8262-abd77ceeb36a, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:18.736208327Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:18.736277957Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 061f99f8-6626-4a81-8262-abd77ceeb36a, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 29db7ccb-90e1-414b-aeee-a9393f639631, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:18.756007475Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:18.756047086Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 29db7ccb-90e1-414b-aeee-a9393f639631, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:18.775193992Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:18.775234455Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d857801a-a95b-4937-a2f0-31ad7c239ac7, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:18.794022791Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:18.794079661Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: d857801a-a95b-4937-a2f0-31ad7c239ac7, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:18.813082137Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:18.813128629Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:18.832592288Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:18.832646111Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 58b140e5-f61e-4410-8851-af929dfd8069, dataset_name: DatasetName { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:18.850930351Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:18.850984259Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 58b140e5-f61e-4410-8851-af929dfd8069, dataset_name: DatasetName { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:18.869703802Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:18.869742653Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, dataset_name: DatasetName { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:18.887923304Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:18.887974427Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, dataset_name: DatasetName { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1741444-7ab7-4888-aa3c-e7b6da3527da, dataset_name: DatasetName { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:18.906608593Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:18.906657888Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: c1741444-7ab7-4888-aa3c-e7b6da3527da, dataset_name: DatasetName { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:18.925390268Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Reading ledger from /pool/int/4576fedf-479b-43b3-8fad-bb8b7ec274f8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:18.925439862Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/d09b3f64-0a36-4052-8233-a94e62e662e2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:18.92565609Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:18.925773946Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:45:19.020304699Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:45:19.02036664Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:45:19.020385222Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:45:19.020393612Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:45:19.02039994Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:45:19.020408313Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:45:19.020416573Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:45:19.020429304Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:45:19.020434151Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:45:19.020440844Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:45:19.020447294Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:45:19.020529979Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:45:19.02053671Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:45:19.02054136Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:45:19.020547698Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:45:19.020560288Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:45:19.020566626Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:45:19.020571707Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:45:19.020576105Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:19.020588986Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"aea4e807-bd57-4097-8608-4714b04e8d5a","remote_addr":"[fd00:1122:3344:101::1]:59148","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb\", TimeNotSynchronized), (\"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf\", TimeNotSynchronized), (\"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2\", TimeNotSynchronized), (\"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8\", TimeNotSynchronized), (\"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d\", TimeNotSynchronized), (\"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d\", TimeNotSynchronized), (\"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069\", TimeNotSynchronized), (\"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015\", TimeNotSynchronized), (\"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451\", TimeNotSynchronized), (\"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da\", TimeNotSynchronized), (\"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65\", TimeNotSynchronized), (\"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a\", TimeNotSynchronized), (\"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8\", TimeNotSynchronized), (\"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631\", TimeNotSynchronized), (\"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300\", TimeNotSynchronized), (\"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7\", TimeNotSynchronized), (\"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83\", TimeNotSynchronized), (\"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1\", TimeNotSynchronized), (\"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\", TimeNotSynchronized)]","latency_us":477456,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:45:19.020763242Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"14.035870625s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"aea4e807-bd57-4097-8608-4714b04e8d5a\", \"content-length\": \"124\", \"date\": \"Mon, 05 Aug 2024 16:45:18 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"aea4e807-bd57-4097-8608-4714b04e8d5a\" }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:23.660353523Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:36117"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:23.660429883Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"1d28d696-6516-4982-b260-a6c50a59cba2","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:36117","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":41,"response_code":"200"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 83a194e2-3464-4a84-ae47-e33a0983bc56, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: fe17df80-010a-4b68-9ebc-6a94437be875, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5b6cee62-004c-4843-9e16-c25d2b0c743b, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6300d678-c957-4259-9a3e-c167ee4d0fb0, kind: Service { id: 5b6cee62-004c-4843-9e16-c25d2b0c743b }, name: Name(\"ntp-5b6cee62-004c-4843-9e16-c25d2b0c743b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 71])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 3c470f26-8422-4139-9f7f-81a0d84becba, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: acae5a41-5da9-420f-9bc8-154e4abdaf87, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } } } }, OmicronZoneConfig { id: eed4ad2f-7765-43e1-8943-ea458ed22e57, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } } } }, OmicronZoneConfig { id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 902ea001-1913-4130-9979-87a8d3db1f53, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8995285b-5cb0-4d3a-b021-8aec05314a65, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: c8555f5b-90c8-4b5b-b73e-5b0ba7ce0695, kind: Service { id: 8995285b-5cb0-4d3a-b021-8aec05314a65 }, name: Name(\"external-dns-8995285b-5cb0-4d3a-b021-8aec05314a65\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 233, 199])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 0a07869d-c9ec-4ade-b114-97ceff54b60e, kind: Service { id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8 }, name: Name(\"external-dns-8a344ff3-4023-4f5e-a29a-3db2dc6980b8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 206, 204])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 866d6328-4ca1-435e-8a98-cd5268df29e8, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 6eedd1d7-7b63-4139-96c3-ad58e9233e7c, kind: Service { id: 866d6328-4ca1-435e-8a98-cd5268df29e8 }, name: Name(\"nexus-866d6328-4ca1-435e-8a98-cd5268df29e8\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 250])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4952ec5d-23ba-48ea-9509-011fcb02adaf, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 38978e97-2109-4173-8624-98de2b729958, kind: Service { id: 4952ec5d-23ba-48ea-9509-011fcb02adaf }, name: Name(\"nexus-4952ec5d-23ba-48ea-9509-011fcb02adaf\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 188])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1c82ddd6-6f5a-4a92-a9b3-fde01c046c42, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 51730dfc-654f-49d5-a0b7-5f1b747fb3be, kind: Service { id: 1c82ddd6-6f5a-4a92-a9b3-fde01c046c42 }, name: Name(\"nexus-1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 28])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 893afdd2-39c6-4f94-b381-facb1a40bbb1, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 061f99f8-6626-4a81-8262-abd77ceeb36a, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 15711896-d3e4-44e3-b330-624f0f5e064d, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: c02f008f-19c7-48c5-93c6-3f776c51e015, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 738351b7-dacb-486a-ab38-cf8aefaef9cb, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 29db7ccb-90e1-414b-aeee-a9393f639631, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } } } }, OmicronZoneConfig { id: d857801a-a95b-4937-a2f0-31ad7c239ac7, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External } } } }, OmicronZoneConfig { id: 58b140e5-f61e-4410-8851-af929dfd8069, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External } } } }, OmicronZoneConfig { id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External } } } }, OmicronZoneConfig { id: c1741444-7ab7-4888-aa3c-e7b6da3527da, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:33.09554392Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83a194e2-3464-4a84-ae47-e33a0983bc56, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:33.095938613Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:33.095947163Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 83a194e2-3464-4a84-ae47-e33a0983bc56, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:33.11625617Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:33.11829654Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe17df80-010a-4b68-9ebc-6a94437be875, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:33.137105788Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:33.137155223Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: fe17df80-010a-4b68-9ebc-6a94437be875, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c470f26-8422-4139-9f7f-81a0d84becba, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:33.155317281Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:33.155367082Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 3c470f26-8422-4139-9f7f-81a0d84becba, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: acae5a41-5da9-420f-9bc8-154e4abdaf87, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:33.173560116Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:33.175985956Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: acae5a41-5da9-420f-9bc8-154e4abdaf87, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eed4ad2f-7765-43e1-8943-ea458ed22e57, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:33.192077692Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:33.192127109Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: eed4ad2f-7765-43e1-8943-ea458ed22e57, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:33.211690445Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:33.211743677Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 902ea001-1913-4130-9979-87a8d3db1f53, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:33.228981789Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:33.229086421Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 902ea001-1913-4130-9979-87a8d3db1f53, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8995285b-5cb0-4d3a-b021-8aec05314a65, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:33.247308046Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:33.247353291Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8995285b-5cb0-4d3a-b021-8aec05314a65, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:33.26463201Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:33.264678539Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 061f99f8-6626-4a81-8262-abd77ceeb36a, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:33.281850717Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:33.281896853Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 061f99f8-6626-4a81-8262-abd77ceeb36a, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 29db7ccb-90e1-414b-aeee-a9393f639631, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:33.302782027Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:33.305119526Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 29db7ccb-90e1-414b-aeee-a9393f639631, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:33.321401007Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:33.321446852Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d857801a-a95b-4937-a2f0-31ad7c239ac7, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:33.338451614Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:33.33849609Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: d857801a-a95b-4937-a2f0-31ad7c239ac7, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:33.358207365Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:33.358255206Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:33.37688194Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:33.376929163Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 58b140e5-f61e-4410-8851-af929dfd8069, dataset_name: DatasetName { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:33.394976905Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:33.395018859Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 58b140e5-f61e-4410-8851-af929dfd8069, dataset_name: DatasetName { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:33.411542114Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:33.411586187Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, dataset_name: DatasetName { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:33.42951318Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:33.429623961Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, dataset_name: DatasetName { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1741444-7ab7-4888-aa3c-e7b6da3527da, dataset_name: DatasetName { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:33.447339964Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:33.44738294Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: c1741444-7ab7-4888-aa3c-e7b6da3527da, dataset_name: DatasetName { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:33.465372668Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Reading ledger from /pool/int/4576fedf-479b-43b3-8fad-bb8b7ec274f8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:33.465430203Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/d09b3f64-0a36-4052-8233-a94e62e662e2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:33.465768099Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:33.465907714Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:45:33.557659155Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:45:33.557736265Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:45:33.557745172Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:45:33.557751715Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:45:33.557765281Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:45:33.557773598Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:45:33.557783423Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:45:33.557792441Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:45:33.557799369Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:45:33.557812842Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:45:33.557819817Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:45:33.557827927Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:45:33.557845673Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:45:33.557864405Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:45:33.557944895Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:45:33.557952574Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:45:33.55796208Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:45:33.557966974Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1"} {"msg":"Zone failed to start","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:45:33.557973189Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3065","zone":"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:33.558047685Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"69737ee5-e729-47d5-b9a3-86fd5759d03f","remote_addr":"[fd00:1122:3344:101::1]:59148","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:902","error_message_external":"Internal Server Error","error_message_internal":"Failed to initialize zones: [(\"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65\", TimeNotSynchronized), (\"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf\", TimeNotSynchronized), (\"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\", TimeNotSynchronized), (\"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d\", TimeNotSynchronized), (\"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015\", TimeNotSynchronized), (\"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb\", TimeNotSynchronized), (\"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631\", TimeNotSynchronized), (\"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83\", TimeNotSynchronized), (\"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2\", TimeNotSynchronized), (\"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d\", TimeNotSynchronized), (\"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069\", TimeNotSynchronized), (\"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451\", TimeNotSynchronized), (\"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7\", TimeNotSynchronized), (\"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300\", TimeNotSynchronized), (\"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da\", TimeNotSynchronized), (\"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8\", TimeNotSynchronized), (\"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8\", TimeNotSynchronized), (\"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1\", TimeNotSynchronized), (\"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a\", TimeNotSynchronized)]","latency_us":461051,"response_code":"500"} {"msg":"failed to initialize Omicron zones","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:45:33.55819838Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:475","retry_after":"15.380709088s","error":"Error Response: status: 500 Internal Server Error; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"69737ee5-e729-47d5-b9a3-86fd5759d03f\", \"content-length\": \"124\", \"date\": \"Mon, 05 Aug 2024 16:45:33 GMT\"}; value: Error { error_code: Some(\"Internal\"), message: \"Internal Server Error\", request_id: \"69737ee5-e729-47d5-b9a3-86fd5759d03f\" }"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:45:36.932512595Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","component":"MetricsManager","sled_id":"3abf3a4f-8820-40af-a01f-4369b42a5529","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): operation timed out\"","delay":"46.079444441s"} {"msg":"attempting to set up sled's Omicron zones: OmicronZonesConfig { generation: Generation(5), zones: [OmicronZoneConfig { id: 83a194e2-3464-4a84-ae47-e33a0983bc56, underlay_address: fd00:1122:3344:1::1, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } }, http_address: [fd00:1122:3344:1::1]:5353, dns_address: [fd00:1122:3344:1::1]:53, gz_address: fd00:1122:3344:1::2, gz_address_index: 0 } }, OmicronZoneConfig { id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, underlay_address: fd00:1122:3344:2::1, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } }, http_address: [fd00:1122:3344:2::1]:5353, dns_address: [fd00:1122:3344:2::1]:53, gz_address: fd00:1122:3344:2::2, gz_address_index: 1 } }, OmicronZoneConfig { id: fe17df80-010a-4b68-9ebc-6a94437be875, underlay_address: fd00:1122:3344:3::1, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: InternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } }, http_address: [fd00:1122:3344:3::1]:5353, dns_address: [fd00:1122:3344:3::1]:53, gz_address: fd00:1122:3344:3::2, gz_address_index: 2 } }, OmicronZoneConfig { id: 5b6cee62-004c-4843-9e16-c25d2b0c743b, underlay_address: fd00:1122:3344:101::1b, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: BoundaryNtp { address: [fd00:1122:3344:101::1b]:123, ntp_servers: [\"0.pool.ntp.org\"], dns_servers: [1.1.1.1, 9.9.9.9], domain: None, nic: NetworkInterface { id: 6300d678-c957-4259-9a3e-c167ee4d0fb0, kind: Service { id: 5b6cee62-004c-4843-9e16-c25d2b0c743b }, name: Name(\"ntp-5b6cee62-004c-4843-9e16-c25d2b0c743b\"), ip: 172.30.3.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 247, 71])), subnet: V4(Ipv4Net { addr: 172.30.3.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, snat_cfg: SourceNatConfig { ip: 10.151.1.169, first_port: 0, last_port: 16383 } } }, OmicronZoneConfig { id: 3c470f26-8422-4139-9f7f-81a0d84becba, underlay_address: fd00:1122:3344:101::3, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::3]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: acae5a41-5da9-420f-9bc8-154e4abdaf87, underlay_address: fd00:1122:3344:101::4, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::4]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } } } }, OmicronZoneConfig { id: eed4ad2f-7765-43e1-8943-ea458ed22e57, underlay_address: fd00:1122:3344:101::5, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::5]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } } } }, OmicronZoneConfig { id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, underlay_address: fd00:1122:3344:101::6, filesystem_pool: Some(ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::6]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 902ea001-1913-4130-9979-87a8d3db1f53, underlay_address: fd00:1122:3344:101::7, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: CockroachDb { address: [fd00:1122:3344:101::7]:32221, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8995285b-5cb0-4d3a-b021-8aec05314a65, underlay_address: fd00:1122:3344:101::8, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } }, http_address: [fd00:1122:3344:101::8]:5353, dns_address: 10.151.1.164:53, nic: NetworkInterface { id: c8555f5b-90c8-4b5b-b73e-5b0ba7ce0695, kind: Service { id: 8995285b-5cb0-4d3a-b021-8aec05314a65 }, name: Name(\"external-dns-8995285b-5cb0-4d3a-b021-8aec05314a65\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 233, 199])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, underlay_address: fd00:1122:3344:101::9, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: ExternalDns { dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } }, http_address: [fd00:1122:3344:101::9]:5353, dns_address: 10.151.1.165:53, nic: NetworkInterface { id: 0a07869d-c9ec-4ade-b114-97ceff54b60e, kind: Service { id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8 }, name: Name(\"external-dns-8a344ff3-4023-4f5e-a29a-3db2dc6980b8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 206, 204])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] } } }, OmicronZoneConfig { id: 866d6328-4ca1-435e-8a98-cd5268df29e8, underlay_address: fd00:1122:3344:101::a, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::a]:12221, external_ip: 10.151.1.166, nic: NetworkInterface { id: 6eedd1d7-7b63-4139-96c3-ad58e9233e7c, kind: Service { id: 866d6328-4ca1-435e-8a98-cd5268df29e8 }, name: Name(\"nexus-866d6328-4ca1-435e-8a98-cd5268df29e8\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 250])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 4952ec5d-23ba-48ea-9509-011fcb02adaf, underlay_address: fd00:1122:3344:101::b, filesystem_pool: Some(ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::b]:12221, external_ip: 10.151.1.167, nic: NetworkInterface { id: 38978e97-2109-4173-8624-98de2b729958, kind: Service { id: 4952ec5d-23ba-48ea-9509-011fcb02adaf }, name: Name(\"nexus-4952ec5d-23ba-48ea-9509-011fcb02adaf\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 188])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 1c82ddd6-6f5a-4a92-a9b3-fde01c046c42, underlay_address: fd00:1122:3344:101::c, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: Nexus { internal_address: [fd00:1122:3344:101::c]:12221, external_ip: 10.151.1.168, nic: NetworkInterface { id: 51730dfc-654f-49d5-a0b7-5f1b747fb3be, kind: Service { id: 1c82ddd6-6f5a-4a92-a9b3-fde01c046c42 }, name: Name(\"nexus-1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 28])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, external_tls: true, external_dns_servers: [1.1.1.1, 9.9.9.9] } }, OmicronZoneConfig { id: 893afdd2-39c6-4f94-b381-facb1a40bbb1, underlay_address: fd00:1122:3344:101::d, filesystem_pool: Some(ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }), zone_type: Oximeter { address: [fd00:1122:3344:101::d]:12223 } }, OmicronZoneConfig { id: 061f99f8-6626-4a81-8262-abd77ceeb36a, underlay_address: fd00:1122:3344:101::e, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: Clickhouse { address: [fd00:1122:3344:101::e]:8123, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 15711896-d3e4-44e3-b330-624f0f5e064d, underlay_address: fd00:1122:3344:101::f, filesystem_pool: Some(ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::f]:17000 } }, OmicronZoneConfig { id: c02f008f-19c7-48c5-93c6-3f776c51e015, underlay_address: fd00:1122:3344:101::10, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::10]:17000 } }, OmicronZoneConfig { id: 738351b7-dacb-486a-ab38-cf8aefaef9cb, underlay_address: fd00:1122:3344:101::11, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: CruciblePantry { address: [fd00:1122:3344:101::11]:17000 } }, OmicronZoneConfig { id: 29db7ccb-90e1-414b-aeee-a9393f639631, underlay_address: fd00:1122:3344:101::12, filesystem_pool: Some(ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::12]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External } } } }, OmicronZoneConfig { id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, underlay_address: fd00:1122:3344:101::13, filesystem_pool: Some(ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::13]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External } } } }, OmicronZoneConfig { id: d857801a-a95b-4937-a2f0-31ad7c239ac7, underlay_address: fd00:1122:3344:101::14, filesystem_pool: Some(ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::14]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External } } } }, OmicronZoneConfig { id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, underlay_address: fd00:1122:3344:101::15, filesystem_pool: Some(ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::15]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, underlay_address: fd00:1122:3344:101::16, filesystem_pool: Some(ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::16]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External } } } }, OmicronZoneConfig { id: 58b140e5-f61e-4410-8851-af929dfd8069, underlay_address: fd00:1122:3344:101::17, filesystem_pool: Some(ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::17]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External } } } }, OmicronZoneConfig { id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, underlay_address: fd00:1122:3344:101::18, filesystem_pool: Some(ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::18]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External } } } }, OmicronZoneConfig { id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, underlay_address: fd00:1122:3344:101::19, filesystem_pool: Some(ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::19]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External } } } }, OmicronZoneConfig { id: c1741444-7ab7-4888-aa3c-e7b6da3527da, underlay_address: fd00:1122:3344:101::1a, filesystem_pool: Some(ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }), zone_type: Crucible { address: [fd00:1122:3344:101::1a]:32345, dataset: OmicronZoneDataset { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External } } } }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:48.981948843Z","hostname":"buskin","pid":1555,"sled_address":"[fd00:1122:3344:101::1]:12345","component":"RSS","file":"sled-agent/src/rack_setup/service.rs:438"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 83a194e2-3464-4a84-ae47-e33a0983bc56, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:48.982439248Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:48.982459058Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 83a194e2-3464-4a84-ae47-e33a0983bc56, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:49.004745758Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:49.007154644Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 5145c3cc-86a8-4ca7-a5dd-d58654518f19, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: fe17df80-010a-4b68-9ebc-6a94437be875, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: InternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:49.025346437Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:49.025412644Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: fe17df80-010a-4b68-9ebc-6a94437be875, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: InternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 3c470f26-8422-4139-9f7f-81a0d84becba, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:49.044346015Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:49.044386496Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 3c470f26-8422-4139-9f7f-81a0d84becba, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: acae5a41-5da9-420f-9bc8-154e4abdaf87, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:49.064484818Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:49.064538388Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: acae5a41-5da9-420f-9bc8-154e4abdaf87, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: eed4ad2f-7765-43e1-8943-ea458ed22e57, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:49.083236142Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:49.083290668Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: eed4ad2f-7765-43e1-8943-ea458ed22e57, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:49.102350583Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:49.102404274Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: b2f11b30-44fe-4b76-98c0-1891b1b0971a, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 902ea001-1913-4130-9979-87a8d3db1f53, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: CockroachDb }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:49.121490884Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:49.121546929Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 902ea001-1913-4130-9979-87a8d3db1f53, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: CockroachDb }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8995285b-5cb0-4d3a-b021-8aec05314a65, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:49.139633653Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:49.139675242Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8995285b-5cb0-4d3a-b021-8aec05314a65, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: ExternalDns }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:49.158893917Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:49.158939612Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: ExternalDns }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 061f99f8-6626-4a81-8262-abd77ceeb36a, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Clickhouse }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:49.178897548Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:49.178951211Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 061f99f8-6626-4a81-8262-abd77ceeb36a, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Clickhouse }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 29db7ccb-90e1-414b-aeee-a9393f639631, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:49.197353381Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:49.199659329Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 29db7ccb-90e1-414b-aeee-a9393f639631, dataset_name: DatasetName { pool_name: ZpoolName { id: bd033efe-47fe-4ba2-8bc4-ba99e628ada7 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:49.217004438Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:49.217046561Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 8029adce-2a1d-48e9-a435-7bcff4f3bb83, dataset_name: DatasetName { pool_name: ZpoolName { id: 1aa678c2-e32d-4428-a115-7e698777172b (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: d857801a-a95b-4937-a2f0-31ad7c239ac7, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:49.235667651Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:49.235719046Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: d857801a-a95b-4937-a2f0-31ad7c239ac7, dataset_name: DatasetName { pool_name: ZpoolName { id: cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:49.255078602Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:49.255119713Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 0d925b8a-ac40-4a91-9334-d6f7d7a29fb2, dataset_name: DatasetName { pool_name: ZpoolName { id: 939962e4-70b1-47be-b2d6-938c818b62c4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:49.274977916Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:49.275021342Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 48f67680-6ea6-4e0c-9968-c45baf393c1d, dataset_name: DatasetName { pool_name: ZpoolName { id: 52c33af8-e5fc-4cb6-93db-7a529f23be77 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 58b140e5-f61e-4410-8851-af929dfd8069, dataset_name: DatasetName { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:49.293515721Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:49.293636197Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 58b140e5-f61e-4410-8851-af929dfd8069, dataset_name: DatasetName { pool_name: ZpoolName { id: cb14544a-cc03-4a04-84a7-da1d51f0a71d (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:49.312480674Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:49.312524812Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: f5fbccc2-b8b1-4dd2-9655-42d226ad2451, dataset_name: DatasetName { pool_name: ZpoolName { id: 2cb76428-77cf-4416-ab3e-f66fc49a03b4 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, dataset_name: DatasetName { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:49.331518429Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:49.331563561Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: 19f0f364-8840-4b44-9eea-e3a6f21b6300, dataset_name: DatasetName { pool_name: ZpoolName { id: 1a42fe8d-3436-46f3-8051-0d2ff2a0be4f (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received NewFilesystem(NewFilesystemRequest { dataset_id: c1741444-7ab7-4888-aa3c-e7b6da3527da, dataset_name: DatasetName { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }, kind: Crucible }, responder: ... })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:49.350543376Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"add_dataset","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:49.350589605Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:772","request":"NewFilesystemRequest { dataset_id: c1741444-7ab7-4888-aa3c-e7b6da3527da, dataset_name: DatasetName { pool_name: ZpoolName { id: 85b774a7-53a2-4f22-a027-1f642ba2d7f9 (zpool), kind: External }, kind: Crucible }, responder: ... }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:49.369392211Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Reading ledger from /pool/int/4576fedf-479b-43b3-8fad-bb8b7ec274f8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:49.369455559Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/d09b3f64-0a36-4052-8233-a94e62e662e2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:49.369653432Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:49.369842749Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:49.460592315Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:260"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:49.475036652Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:260"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:49.475365529Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:49.475441609Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:49.486975633Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:260"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:49.545218098Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:49.54534663Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:49.545352791Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:49.545359962Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:49.545391087Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:49.718951635Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:260"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:49.719229149Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:49.719264156Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:49.71927835Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:49.719287809Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Resolving switch zone addresses in DNS","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:49.743665515Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:260"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:49.743965838Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:49.743987982Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:49.745319415Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Configuring new Omicron zone: oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:49.745586232Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:49.750017461Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:49.754675343Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:49.759629786Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:49.764179681Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:49.769084444Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:49.773699491Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:49.912728563Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:49.917638935Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:49.922471302Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:49.926876722Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:49.931180881Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:49.93571083Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:49.940877404Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Configuring new Omicron zone: oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:49.945940566Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:49.952030341Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:49.95755205Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:49.963270565Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:49.969610007Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:49.974906235Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:49.981575521Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:50.024121957Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:277","addr":"fd00:1122:3344:101::2"} {"msg":"Installing Omicron zone: oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:50.024717632Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:50.031938992Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:50.038163391Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:50.044899908Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:50.052273189Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:50.059506662Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:50.067528898Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:50.217315693Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:277","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:50.293547928Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:277","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:50.297775419Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:302"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:50.297792144Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:194","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:50.351115732Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:426","port":"Port { inner: PortInner(PortData { name: \"opte1\", ip: 172.30.2.7, mac: MacAddr6([168, 64, 37, 255, 202, 28]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:50.351142732Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:1210","zone_type":"nexus"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:50.405050742Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:277","addr":"fd00:1122:3344:101::2"} {"msg":"Querying MGS to determine switch location","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:50.486425923Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:277","addr":"fd00:1122:3344:101::2"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:50.492400088Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:302"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:50.492414628Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:194","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:50.569258899Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:426","port":"Port { inner: PortInner(PortData { name: \"opte2\", ip: 172.30.1.5, mac: MacAddr6([168, 64, 37, 255, 233, 199]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:50.569278999Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:1210","zone_type":"external_dns"} {"msg":"no nat entry found for: 10.151.1.168","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:50.569291543Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.168","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:50.569311428Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:50.569344766Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:302"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:50.569352313Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:194","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:50.627980959Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:426","port":"Port { inner: PortInner(PortData { name: \"opte3\", ip: 172.30.1.6, mac: MacAddr6([168, 64, 37, 255, 206, 204]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.1.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:50.628007941Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:1210","zone_type":"external_dns"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:50.630331863Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:302"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:50.630345579Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:194","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:50.680172719Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:426","port":"Port { inner: PortInner(PortData { name: \"opte4\", ip: 172.30.2.5, mac: MacAddr6([168, 64, 37, 255, 210, 250]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:50.680198416Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:1210","zone_type":"nexus"} {"msg":"Found Switch0 at fd00:1122:3344:101::2","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:50.682137084Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:302"} {"msg":"Successfully looked up all expected switch zone underlay addresses","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:50.682152103Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/bootstrap/early_networking.rs:194","addrs":"{Switch0: fd00:1122:3344:101::2}"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:50.73151389Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:426","port":"Port { inner: PortInner(PortData { name: \"opte5\", ip: 172.30.2.6, mac: MacAddr6([168, 64, 37, 255, 216, 188]), slot: 0, vni: Vni { inner: 100 }, subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.2.1 } }) }"} {"msg":"creating NAT entry for service","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:50.731533494Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:1210","zone_type":"nexus"} {"msg":"creation of nat entry successful for: 10.151.1.168","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:50.731845621Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:50.733782685Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"no nat entry found for: 10.151.1.164","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:50.816396259Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.164","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:50.816496283Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"no nat entry found for: 10.151.1.167","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:50.816961022Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.167","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:50.816993338Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"no nat entry found for: 10.151.1.165","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:50.817815776Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.165","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:50.817832399Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"creation of nat entry successful for: 10.151.1.167","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:50.818560549Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:50.821153487Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"creation of nat entry successful for: 10.151.1.164","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:50.821177562Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"creation of nat entry successful for: 10.151.1.165","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:50.826471552Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"no nat entry found for: 10.151.1.166","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:50.830070894Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:83"} {"msg":"creating nat entry for: 10.151.1.166","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:50.830082173Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:90"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:50.830618291Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:50.84068844Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Configuring new Omicron zone: oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:50.884757801Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"creation of nat entry successful for: 10.151.1.166","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:50.976061908Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"clients/dpd-client/src/lib.rs:118"} {"msg":"Configuring new Omicron zone: oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:50.978625306Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:50.998971515Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Installing Omicron zone: oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:50.999127743Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Configuring new Omicron zone: oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:51.012325405Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Configuring new Omicron zone: oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:51.027664787Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:51.227138371Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:51.252968214Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Installing Omicron zone: oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:51.270585132Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Configuring new Omicron zone: oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:51.294696948Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:51.350726451Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:53.09658671Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:53.09684208Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:472"} {"msg":"Profile for oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:53.250879898Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015 zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:53.251399334Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:472"} {"msg":"Profile for oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:53.443264033Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631 zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:53.443440333Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:472"} {"msg":"Profile for oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:53.453445528Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:53.453598445Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:472"} {"msg":"Profile for oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:53.520685111Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7 zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:53.520975376Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:472"} {"msg":"Profile for oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:53.536513899Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83 zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:53.536711172Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:472"} {"msg":"Profile for oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:53.569067376Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069 zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:53.571550642Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:472"} {"msg":"Profile for oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:53.597502485Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:53.60183354Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:53.602049094Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:472"} {"msg":"Profile for oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:53.633687563Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2 zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:53.639903938Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:472"} {"msg":"Booting oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1 zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:53.660508364Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:472"} {"msg":"wait for service Some(\"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb\") failed: Property not found. retry in 54.049664ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:45:54.930798692Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:54.932567745Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:54.934613166Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:45:54.935144287Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:472"} {"msg":"wait for service Some(\"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015\") failed: Property not found. retry in 29.995804ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:45:59.09801437Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb\") failed: Property not found. retry in 102.725107ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:00.169324338Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:00.190626322Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300 zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:00.191886503Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:472"} {"msg":"wait for service Some(\"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d\") failed: Property not found. retry in 68.059496ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:00.304028867Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015\") failed: Property not found. retry in 85.269964ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:00.525904343Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb\") failed: Property not found. retry in 223.644782ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:00.722566828Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:00.906045199Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:37841"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:00.908273706Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"3150380f-9718-4bfb-9c2b-8939b99047d0","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:37841","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":1201,"response_code":"200"} {"msg":"wait for service Some(\"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631\") failed: Property not found. retry in 32.246802ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:00.912669981Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83\") failed: Property not found. retry in 63.387478ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:00.988673428Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069\") failed: Property not found. retry in 53.76981ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:01.063536625Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Booting oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451 zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:01.063633967Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:472"} {"msg":"wait for service Some(\"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7\") failed: Property not found. retry in 44.93725ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:01.134034498Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d\") failed: Property not found. retry in 109.365207ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:01.164188522Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015\") failed: Property not found. retry in 250.213069ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:01.200932409Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb\") failed: Property not found. retry in 440.723297ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:01.33583481Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631\") failed: Property not found. retry in 99.052458ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:01.3956537Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83\") failed: Property not found. retry in 104.954736ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:01.430521125Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d\") failed: Property not found. retry in 55.704472ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:01.458446142Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069\") failed: Property not found. retry in 105.429152ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:01.513907188Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da\") failed: Property not found. retry in 72.686803ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:01.535073789Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7\") failed: Property not found. retry in 87.236334ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:01.556181318Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2\") failed: Property not found. retry in 52.785714ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:01.573682206Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d\") failed: Property not found. retry in 238.346862ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:02.181700908Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1\") failed: Property not found. retry in 35.519733ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:02.356932557Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015\") failed: Property not found. retry in 236.267927ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:02.375070526Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb\") failed: Property not found. retry in 563.669882ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:02.923921967Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631\") failed: Property not found. retry in 272.444074ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:03.234978443Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83\") failed: Property not found. retry in 247.513277ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:03.252074265Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d\") failed: Property not found. retry in 81.433634ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:03.269651225Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069\") failed: Property not found. retry in 113.634146ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:03.284997366Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451\") failed: Property not found. retry in 54.188475ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:03.299645326Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300\") failed: Property not found. retry in 47.593918ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:03.320153466Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da\") failed: Property not found. retry in 63.700525ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:03.340085086Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7\") failed: Property not found. retry in 296.585249ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:03.355825085Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2\") failed: Property not found. retry in 142.052806ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:03.375584892Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d\") failed: Property not found. retry in 405.330203ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:03.394686557Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1\") failed: Property not found. retry in 89.503664ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:03.410119761Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015\") failed: Property not found. retry in 665.414831ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:03.421935192Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d\") failed: Property not found. retry in 185.974552ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:03.434854033Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069\") failed: Property not found. retry in 337.070339ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:03.44584115Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451\") failed: Property not found. retry in 144.65644ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:03.455529365Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300\") failed: Property not found. retry in 77.662037ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:03.465351483Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da\") failed: Property not found. retry in 284.920688ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:03.476452792Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb\") failed: Property not found. retry in 831.960904ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:03.502146161Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1\") failed: Property not found. retry in 298.068101ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:03.513542175Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83\") failed: Property not found. retry in 238.276856ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:03.523976863Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631\") failed: Property not found. retry in 416.306213ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:03.53785005Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2\") failed: Property not found. retry in 183.161355ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:03.552326976Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300\") failed: Property not found. retry in 265.838338ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:03.564083606Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451\") failed: Property not found. retry in 262.661049ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:03.616574012Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d\") failed: Property not found. retry in 229.975198ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:03.636014604Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7\") failed: Property not found. retry in 418.73667ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:03.66628507Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2\") failed: Property not found. retry in 581.669305ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:03.754870081Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da\") failed: Property not found. retry in 254.709467ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:03.778458523Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83\") failed: Property not found. retry in 790.715713ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:03.797486388Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069\") failed: Property not found. retry in 553.128675ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:03.814356166Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1\") failed: Property not found. retry in 547.697325ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:03.830215616Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d\") failed: Property not found. retry in 661.98814ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:03.847711255Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300\") failed: Property not found. retry in 239.934223ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:03.866250011Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d\") failed: Property not found. retry in 678.281081ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:03.885979717Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451\") failed: Property not found. retry in 346.257073ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:03.902103472Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631\") failed: Property not found. retry in 662.981339ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:03.969423741Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:04.009354332Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8 zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:04.009687207Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:472"} {"msg":"wait for service Some(\"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da\") failed: Property not found. retry in 975.960684ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:04.051394109Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7\") failed: Property not found. retry in 469.807383ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:04.103941795Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015\") failed: Property not found. retry in 1.216768402s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:04.117510597Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300\") failed: Property not found. retry in 1.074222781s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:04.138595296Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451\") failed: Property not found. retry in 589.722914ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:04.265465161Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8\") failed: Property not found. retry in 27.575233ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:04.278402171Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8\") failed: Property not found. retry in 64.304588ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:04.324447136Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb\") failed: Property not found. retry in 1.406746961s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:04.353539373Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2\") failed: Property not found. retry in 842.76938ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:04.371631991Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069\") failed: Property not found. retry in 1.391522665s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:04.391832246Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1\") failed: Property not found. retry in 660.417478ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:04.413840471Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8\") failed: Property not found. retry in 244.106689ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:04.430540376Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d\") failed: Property not found. retry in 641.405929ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:04.530235547Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d\") failed: Property not found. retry in 754.67835ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:04.58382403Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7\") failed: Property not found. retry in 1.051342392s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:04.608610193Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83\") failed: Property not found. retry in 1.009569103s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:04.63833222Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631\") failed: Property not found. retry in 1.006487994s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:04.662206567Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8\") failed: Property not found. retry in 565.9474ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:04.698343093Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451\") failed: Property not found. retry in 1.362384841s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:04.870938688Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da\") failed: Property not found. retry in 1.212943843s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:05.042599226Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1\") failed: Property not found. retry in 1.396750098s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:05.086087998Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d\") failed: Property not found. retry in 919.159478ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:05.183860865Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300\") failed: Property not found. retry in 617.142838ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:05.226912169Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2\") failed: Property not found. retry in 1.431828929s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:05.237997519Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8\") failed: Property not found. retry in 585.662782ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:05.275869015Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015\") failed: Property not found. retry in 837.405844ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:05.347237222Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d\") failed: Property not found. retry in 979.603639ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:05.360165413Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:05.638280756Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:05.63855807Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:472"} {"msg":"Profile for oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:05.645320546Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Profile for oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:05.646037758Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42 zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:05.646227491Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:472"} {"msg":"wait for service Some(\"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83\") failed: Property not found. retry in 1.426962537s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:05.662966407Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Booting oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8 zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:05.663226671Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:472"} {"msg":"wait for service Some(\"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7\") failed: Property not found. retry in 992.133089ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:05.680054618Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631\") failed: Property not found. retry in 1.162271041s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:05.690174431Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb\") failed: Property not found. retry in 864.16091ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:05.849365596Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069\") failed: Property not found. retry in 1.246929844s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:05.860026128Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300\") failed: Property not found. retry in 809.89581ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:05.870482806Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8\") failed: Property not found. retry in 61.764374ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:05.878537074Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf\") failed: Property not found. retry in 29.804225ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:05.886099563Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\") failed: Property not found. retry in 71.910708ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:05.89588035Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8\") failed: Property not found. retry in 755.528258ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:05.907252224Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf\") failed: Property not found. retry in 147.769337ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:05.925033757Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8\") failed: Property not found. retry in 102.007386ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:05.949960992Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\") failed: Property not found. retry in 138.051814ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:05.978945897Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8\") failed: Property not found. retry in 287.192137ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:06.062291278Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf\") failed: Property not found. retry in 119.069442ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:06.086213386Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d\") failed: Property not found. retry in 1.113158796s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:06.114601441Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\") failed: Property not found. retry in 196.354594ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:06.130149016Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015\") failed: Property not found. retry in 593.156118ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:06.196271154Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf\") failed: Property not found. retry in 249.415189ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:06.216869085Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451\") failed: Property not found. retry in 771.308838ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:06.501119944Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\") failed: Property not found. retry in 595.518301ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:06.640659623Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1\") failed: Property not found. retry in 1.172735292s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:06.682493967Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8\") failed: Property not found. retry in 749.45941ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:06.717941521Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb\") failed: Property not found. retry in 987.709553ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:06.748629025Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d\") failed: Property not found. retry in 529.807254ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:06.759169025Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300\") failed: Property not found. retry in 906.251316ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:06.769207436Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da\") failed: Property not found. retry in 538.434642ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:06.77943571Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8\") failed: Property not found. retry in 599.13461ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:06.793157755Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7\") failed: Property not found. retry in 1.24121913s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:06.805724809Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2\") failed: Property not found. retry in 818.530437ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:06.817061772Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf\") failed: Property not found. retry in 705.701305ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:06.827861516Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015\") failed: Property not found. retry in 995.001332ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:06.83980488Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631\") failed: Property not found. retry in 904.101238ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:06.866706039Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83\") failed: Property not found. retry in 962.233139ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:07.104629277Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069\") failed: Property not found. retry in 701.616375ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:07.11826515Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d\") failed: Property not found. retry in 777.935068ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:07.24020082Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\") failed: Property not found. retry in 1.0537227s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:07.250072158Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451\") failed: Property not found. retry in 1.457969697s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:07.28416779Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d\") failed: Property not found. retry in 699.93187ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:07.302026595Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da\") failed: Property not found. retry in 1.427347132s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:07.33105425Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8\") failed: Property not found. retry in 1.069317264s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:07.476162705Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8\") failed: Property not found. retry in 946.060841ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:07.486825527Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf\") failed: Property not found. retry in 1.160570434s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:07.546266901Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2\") failed: Property not found. retry in 531.074992ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:07.648488134Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300\") failed: Property not found. retry in 742.521119ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:07.686732006Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb\") failed: Property not found. retry in 900.796042ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:07.748426509Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631\") failed: Property not found. retry in 619.747711ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:08.211013029Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83\") failed: Property not found. retry in 874.291676ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:08.219057826Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d\") failed: Property not found. retry in 713.435835ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:08.226346521Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069\") failed: Property not found. retry in 1.098490243s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:08.232535626Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7\") failed: Property not found. retry in 562.030555ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:08.238742532Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2\") failed: Property not found. retry in 707.013402ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:08.245798493Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d\") failed: Property not found. retry in 1.461419908s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:08.252565378Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1\") failed: Property not found. retry in 1.195002182s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:08.261384838Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015\") failed: Property not found. retry in 1.293793382s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:08.269717821Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300\") failed: Property not found. retry in 879.320129ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:08.479990427Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\") failed: Property not found. retry in 717.771322ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:08.4889191Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8\") failed: Property not found. retry in 1.32602712s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:08.495917859Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:08.497854812Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:08.498374491Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:472"} {"msg":"wait for service Some(\"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8\") failed: Property not found. retry in 1.082215582s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:08.553779315Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb\") failed: Property not found. retry in 1.351057199s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:08.658161455Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf\") failed: Property not found. retry in 809.159015ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:08.716582712Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451\") failed: Property not found. retry in 719.950018ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:08.75048522Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da\") failed: Property not found. retry in 623.827575ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:08.766839289Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7\") failed: Property not found. retry in 1.082125757s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:08.809868463Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631\") failed: Property not found. retry in 1.453831759s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:08.83958922Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a\") failed: Property not found. retry in 56.978128ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:08.868395992Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a\") failed: Property not found. retry in 110.714032ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:08.933605894Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d\") failed: Property not found. retry in 658.574027ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:08.948532479Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2\") failed: Property not found. retry in 537.073937ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:08.960758557Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a\") failed: Property not found. retry in 295.714363ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:09.052049654Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83\") failed: Property not found. retry in 1.027623289s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:09.101869864Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\") failed: Property not found. retry in 959.61914ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:09.216742674Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069\") failed: Property not found. retry in 1.360645786s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:09.339991206Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a\") failed: Property not found. retry in 258.903855ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:09.35607127Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300\") failed: Property not found. retry in 879.969348ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:09.367843341Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da\") failed: Property not found. retry in 637.440379ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:09.399725385Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1\") failed: Property not found. retry in 1.081082694s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:09.469746719Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451\") failed: Property not found. retry in 1.151174811s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:09.482792912Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2\") failed: Property not found. retry in 1.404238864s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:09.511065734Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf\") failed: Property not found. retry in 1.289180666s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:09.538661185Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015\") failed: Property not found. retry in 1.235261758s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:09.578050586Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d\") failed: Property not found. retry in 549.648292ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:09.620099273Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a\") failed: Property not found. retry in 951.858985ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:09.632335472Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8\") failed: Property not found. retry in 973.413074ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:09.648304224Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d\") failed: Property not found. retry in 1.052676908s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:09.725888758Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8\") failed: Property not found. retry in 1.106320683s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:09.833544094Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7\") failed: Property not found. retry in 1.263266409s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:09.905378819Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb\") failed: Property not found. retry in 665.535389ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:10.022790892Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da\") failed: Property not found. retry in 890.9809ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:10.050726089Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83\") failed: Property not found. retry in 679.833069ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:10.138852172Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d\") failed: Property not found. retry in 936.041921ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:10.178817733Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\") failed: Property not found. retry in 1.464961809s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:10.231315519Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300\") failed: Property not found. retry in 1.158379503s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:10.256957697Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631\") failed: Property not found. retry in 529.036754ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:10.301412485Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1\") failed: Property not found. retry in 983.15834ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:10.560566455Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a\") failed: Property not found. retry in 823.657352ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:10.593342194Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8\") failed: Property not found. retry in 829.828817ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:10.630493797Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451\") failed: Property not found. retry in 958.648153ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:10.642719316Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb\") failed: Property not found. retry in 811.774345ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:10.697302797Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069\") failed: Property not found. retry in 960.407462ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:10.70980808Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d\") failed: Property not found. retry in 767.16051ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:10.787847561Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015\") failed: Property not found. retry in 1.215645852s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:10.821847581Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83\") failed: Property not found. retry in 1.286734092s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:10.830850095Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631\") failed: Property not found. retry in 570.130839ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:10.839080025Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf\") failed: Property not found. retry in 766.823499ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:10.846788837Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8\") failed: Property not found. retry in 1.159893481s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:11.025815011Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da\") failed: Property not found. retry in 844.825453ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:11.075950866Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2\") failed: Property not found. retry in 726.585047ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:11.147628118Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d\") failed: Property not found. retry in 1.27063516s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:11.198375994Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7\") failed: Property not found. retry in 814.596329ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:11.565131639Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d\") failed: Property not found. retry in 1.132887289s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:11.610533813Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1\") failed: Property not found. retry in 1.132179173s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:11.750806346Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a\") failed: Property not found. retry in 647.455545ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:11.797580064Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb\") failed: Property not found. retry in 574.019624ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:12.081620719Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631\") failed: Property not found. retry in 841.536718ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:12.199747144Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83\") failed: Property not found. retry in 777.012962ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:12.365177716Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069\") failed: Property not found. retry in 1.366685835s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:12.508517621Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451\") failed: Property not found. retry in 802.588514ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:12.632308272Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300\") failed: Property not found. retry in 1.372439318s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:12.644054652Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da\") failed: Property not found. retry in 823.557388ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:12.666460284Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8\") failed: Property not found. retry in 596.575708ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:12.698420089Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7\") failed: Property not found. retry in 578.79616ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:12.710376273Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2\") failed: Property not found. retry in 638.059813ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:12.758777899Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf\") failed: Property not found. retry in 1.213590063s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:12.805973724Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d\") failed: Property not found. retry in 745.049475ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:12.850812559Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\") failed: Property not found. retry in 1.441450282s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:12.919175275Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1\") failed: Property not found. retry in 1.335298657s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:12.966142326Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015\") failed: Property not found. retry in 1.33788936s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:13.011031432Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Profile for oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65:\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:13.011707793Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"wait for service Some(\"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8\") failed: Property not found. retry in 767.767095ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:13.056907824Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a\") failed: Property not found. retry in 1.373752933s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:13.089038012Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb\") failed: Property not found. retry in 1.209784242s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:13.123951418Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631\") failed: Property not found. retry in 574.621239ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:13.135767189Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d\") failed: Property not found. retry in 1.189748832s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:13.149550384Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Booting oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65 zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:13.149708877Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65","component":"ServiceManager","file":"illumos-utils/src/running_zone.rs:472"} {"msg":"wait for service Some(\"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83\") failed: Property not found. retry in 701.989758ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:13.171433748Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7\") failed: Property not found. retry in 801.618713ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:13.300950607Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8\") failed: Property not found. retry in 1.46047574s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:13.313490321Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65\") failed: Property not found. retry in 44.111472ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:13.388695043Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2\") failed: Property not found. retry in 1.281072482s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:13.413258933Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65\") failed: Property not found. retry in 100.418621ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:13.451537255Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451\") failed: Property not found. retry in 664.705953ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:13.465319941Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da\") failed: Property not found. retry in 1.493094619s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:13.500277727Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65\") failed: Property not found. retry in 203.917179ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:13.562288174Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d\") failed: Property not found. retry in 1.233753766s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:13.604721307Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631\") failed: Property not found. retry in 663.332208ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:13.720816928Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65\") failed: Property not found. retry in 325.821283ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:13.776724792Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8\") failed: Property not found. retry in 1.291128782s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:13.835291031Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83\") failed: Property not found. retry in 986.89443ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:13.882985604Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069\") failed: Property not found. retry in 883.254041ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:13.891330616Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300\") failed: Property not found. retry in 764.683115ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:14.025980823Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf\") failed: Property not found. retry in 1.253050717s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:14.034438727Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65\") failed: Property not found. retry in 459.513442ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:14.112928396Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7\") failed: Property not found. retry in 1.375196395s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:14.121459692Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451\") failed: Property not found. retry in 934.886293ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:14.138398451Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1\") failed: Property not found. retry in 1.193627024s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:14.310413795Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb\") failed: Property not found. retry in 786.387762ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:14.344070839Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d\") failed: Property not found. retry in 658.084726ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:14.353750874Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015\") failed: Property not found. retry in 969.789439ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:14.364889054Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\") failed: Property not found. retry in 782.459381ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:14.37581832Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631\") failed: Property not found. retry in 817.845242ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:14.393903899Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a\") failed: Property not found. retry in 1.12988078s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:14.473323604Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65\") failed: Property not found. retry in 522.727209ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:14.582142009Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2\") failed: Property not found. retry in 1.103538036s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:14.702309009Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8\") failed: Property not found. retry in 524.256118ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:14.783167518Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069\") failed: Property not found. retry in 907.752662ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:14.790975611Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300\") failed: Property not found. retry in 1.378934544s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:14.798941627Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d\") failed: Property not found. retry in 545.356006ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:14.847462145Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83\") failed: Property not found. retry in 888.377632ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:14.878313401Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da\") failed: Property not found. retry in 724.433731ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:15.004770828Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d\") failed: Property not found. retry in 516.68633ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:15.022297657Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451\") failed: Property not found. retry in 1.13480986s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:15.083911008Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65\") failed: Property not found. retry in 1.188620425s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:15.114455529Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8\") failed: Property not found. retry in 848.958873ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:15.135998306Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb\") failed: Property not found. retry in 1.185622753s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:15.144063794Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\") failed: Property not found. retry in 1.058158921s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:15.166994655Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631\") failed: Property not found. retry in 581.947302ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:15.220483778Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf\") failed: Property not found. retry in 1.414529511s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:15.296687347Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8\") failed: Property not found. retry in 628.548151ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:15.316595537Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015\") failed: Property not found. retry in 1.382095136s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:15.34608549Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d\") failed: Property not found. retry in 1.396459598s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:15.401733499Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7\") failed: Property not found. retry in 1.225663739s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:15.508781675Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1\") failed: Property not found. retry in 894.722642ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:15.518236186Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d\") failed: Property not found. retry in 1.341417838s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:15.547304062Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a\") failed: Property not found. retry in 1.413315429s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:15.614913645Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069\") failed: Property not found. retry in 807.002476ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:15.708371729Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da\") failed: Property not found. retry in 636.96929ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:15.7383436Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83\") failed: Property not found. retry in 634.958691ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:15.776232417Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631\") failed: Property not found. retry in 514.190236ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:15.81104017Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2\") failed: Property not found. retry in 963.353281ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:15.819189449Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8\") failed: Property not found. retry in 1.449713179s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:15.956005063Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8\") failed: Property not found. retry in 1.493563491s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:15.994775745Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300\") failed: Property not found. retry in 851.927892ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:16.187860168Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451\") failed: Property not found. retry in 1.487281148s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:16.22843552Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\") failed: Property not found. retry in 699.052716ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:16.236767582Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65\") failed: Property not found. retry in 1.240356763s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:16.313806746Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631\") failed: Property not found. retry in 1.214463379s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:16.335408685Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb\") failed: Property not found. retry in 540.800493ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:16.345066889Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da\") failed: Property not found. retry in 1.084986978s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:16.385823465Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83\") failed: Property not found. retry in 1.384564782s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:16.420783661Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1\") failed: Property not found. retry in 1.195954844s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:16.430059089Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069\") failed: Property not found. retry in 1.428631706s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:16.530529933Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf\") failed: Property not found. retry in 717.93903ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:16.723962272Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015\") failed: Property not found. retry in 1.429260039s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:16.74264447Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7\") failed: Property not found. retry in 957.898844ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:16.755230905Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2\") failed: Property not found. retry in 1.457887502s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:16.797955644Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d\") failed: Property not found. retry in 1.050942502s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:16.811845181Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb\") failed: Property not found. retry in 932.47802ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:16.899608498Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d\") failed: Property not found. retry in 531.428139ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:16.913851151Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\") failed: Property not found. retry in 888.439994ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:16.948620024Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a\") failed: Property not found. retry in 904.5932ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:17.041510688Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300\") failed: Property not found. retry in 749.058483ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:17.053429771Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8\") failed: Property not found. retry in 1.274763288s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:17.417142394Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf\") failed: Property not found. retry in 856.102219ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:17.453677777Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d\") failed: Property not found. retry in 1.402900706s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:17.464348109Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da\") failed: Property not found. retry in 1.182215481s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:17.484602925Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8\") failed: Property not found. retry in 639.764301ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:17.49780378Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631\") failed: Property not found. retry in 826.014779ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:17.562450351Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65\") failed: Property not found. retry in 865.978194ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:17.57231927Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1\") failed: Property not found. retry in 741.200994ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:17.639727579Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7\") failed: Property not found. retry in 717.107003ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:17.731643847Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451\") failed: Property not found. retry in 914.886296ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:17.744913443Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300\") failed: Property not found. retry in 745.319253ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:17.819518095Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83\") failed: Property not found. retry in 769.722777ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:17.830165667Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb\") failed: Property not found. retry in 1.364703415s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:17.845675982Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\") failed: Property not found. retry in 988.934834ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:17.85438585Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d\") failed: Property not found. retry in 1.36672005s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:17.873782411Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a\") failed: Property not found. retry in 1.185431347s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:17.958490706Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069\") failed: Property not found. retry in 1.423363704s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:17.969685627Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8\") failed: Property not found. retry in 763.46773ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:18.149858759Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015\") failed: Property not found. retry in 1.417375208s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:18.186627186Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2\") failed: Property not found. retry in 666.469597ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:18.266159703Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf\") failed: Property not found. retry in 1.061981185s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:18.322072076Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1\") failed: Property not found. retry in 781.450853ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:18.39325421Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631\") failed: Property not found. retry in 1.331357579s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:18.406244342Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65\") failed: Property not found. retry in 692.156821ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:18.452780135Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7\") failed: Property not found. retry in 1.388676427s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:18.467361324Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300\") failed: Property not found. retry in 633.134121ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:18.577935579Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83\") failed: Property not found. retry in 1.063747287s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:18.611499312Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451\") failed: Property not found. retry in 1.260851761s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:18.670160082Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da\") failed: Property not found. retry in 1.462154192s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:18.679723375Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8\") failed: Property not found. retry in 1.063535673s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:18.702700867Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\") failed: Property not found. retry in 1.052479997s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:18.857734701Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d\") failed: Property not found. retry in 995.445115ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:18.878654188Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8\") failed: Property not found. retry in 1.022656339s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:18.924765474Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2\") failed: Property not found. retry in 718.504359ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:18.945899355Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a\") failed: Property not found. retry in 793.86676ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:19.159270883Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65\") failed: Property not found. retry in 566.379828ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:19.175198065Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1\") failed: Property not found. retry in 1.13998943s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:19.193820703Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb\") failed: Property not found. retry in 1.275136395s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:19.226931847Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300\") failed: Property not found. retry in 1.091742577s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:19.241863372Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d\") failed: Property not found. retry in 1.322055677s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:19.257911525Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf\") failed: Property not found. retry in 812.822675ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:19.399128374Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069\") failed: Property not found. retry in 1.149948708s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:19.409834776Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015\") failed: Property not found. retry in 735.582906ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:19.617096771Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2\") failed: Property not found. retry in 1.00308478s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:19.676958671Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83\") failed: Property not found. retry in 1.0639422s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:19.689459355Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631\") failed: Property not found. retry in 1.483897808s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:19.749508779Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65\") failed: Property not found. retry in 1.42237719s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:19.760229752Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8\") failed: Property not found. retry in 1.26778368s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:19.778802268Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7\") failed: Property not found. retry in 563.800479ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:19.869728449Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d\") failed: Property not found. retry in 695.100052ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:19.884638003Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\") failed: Property not found. retry in 612.565246ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:19.920899232Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451\") failed: Property not found. retry in 1.338915523s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:19.944069517Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8\") failed: Property not found. retry in 679.878591ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:19.961527015Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a\") failed: Property not found. retry in 717.721195ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:19.97530402Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da\") failed: Property not found. retry in 828.888732ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:20.154057268Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf\") failed: Property not found. retry in 559.794319ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:20.225752011Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1\") failed: Property not found. retry in 885.785219ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:20.348555355Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300\") failed: Property not found. retry in 650.015616ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:20.35942541Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015\") failed: Property not found. retry in 1.493059892s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:20.373380348Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7\") failed: Property not found. retry in 790.800348ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:20.443723208Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb\") failed: Property not found. retry in 1.08442228s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:20.514885822Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\") failed: Property not found. retry in 664.741362ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:20.546937048Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069\") failed: Property not found. retry in 1.378117357s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:20.575423253Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d\") failed: Property not found. retry in 658.969822ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:20.590557211Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d\") failed: Property not found. retry in 1.451688052s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:20.599974292Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8\") failed: Property not found. retry in 520.969468ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:20.658373558Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2\") failed: Property not found. retry in 591.116015ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:20.693392755Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a\") failed: Property not found. retry in 1.19193164s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:20.703608499Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83\") failed: Property not found. retry in 881.717369ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:20.767412887Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf\") failed: Property not found. retry in 1.414747353s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:20.797252436Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da\") failed: Property not found. retry in 541.590309ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:20.99684064Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300\") failed: Property not found. retry in 708.672423ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:21.024821157Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8\") failed: Property not found. retry in 1.11896247s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:21.065002223Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8\") failed: Property not found. retry in 1.406907141s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:21.193757069Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65\") failed: Property not found. retry in 1.183675482s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:21.208323747Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\") failed: Property not found. retry in 1.39317182s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:21.226222572Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631\") failed: Property not found. retry in 710.580722ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:21.244928271Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1\") failed: Property not found. retry in 1.314509011s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:21.255960339Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7\") failed: Property not found. retry in 768.492737ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:21.268630835Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d\") failed: Property not found. retry in 921.072844ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:21.279609973Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451\") failed: Property not found. retry in 965.578863ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:21.293023982Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2\") failed: Property not found. retry in 1.246690013s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:21.305361492Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da\") failed: Property not found. retry in 1.078685702s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:21.549272581Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb\") failed: Property not found. retry in 923.111793ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:21.609835594Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83\") failed: Property not found. retry in 755.808918ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:21.660286454Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300\") failed: Property not found. retry in 1.15758311s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:21.746102718Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015\") failed: Property not found. retry in 1.321754029s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:21.87582946Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a\") failed: Property not found. retry in 1.352798731s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:21.905642389Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069\") failed: Property not found. retry in 719.074246ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:21.963028787Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631\") failed: Property not found. retry in 1.491803865s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:21.971308308Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7\") failed: Property not found. retry in 1.345168387s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:22.046838666Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8\") failed: Property not found. retry in 745.514708ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:22.195134155Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d\") failed: Property not found. retry in 947.372926ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:22.211155569Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf\") failed: Property not found. retry in 1.363076282s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:22.226431379Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451\") failed: Property not found. retry in 1.175814193s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:22.268144311Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65\") failed: Property not found. retry in 814.403688ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:22.404483946Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83\") failed: Property not found. retry in 915.480197ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:22.425777789Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2\") failed: Property not found. retry in 1.103925563s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:22.564309881Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1\") failed: Property not found. retry in 753.489888ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:22.579700793Z","hostname":"buskin","pid":1555,"zone":"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8\") failed: Property not found. retry in 1.258726544s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:22.613716244Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\") failed: Property not found. retry in 655.698972ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:22.631856103Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069\") failed: Property not found. retry in 767.259643ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:22.69381933Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300\") failed: Property not found. retry in 1.379343321s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:22.915171314Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8\") failed: Property not found. retry in 1.098760675s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:22.951589316Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65\") failed: Property not found. retry in 820.986561ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:23.519927918Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a\") failed: Property not found. retry in 1.248638652s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:23.527427166Z","hostname":"buskin","pid":1555,"zone":"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631\") failed: Property not found. retry in 550.665064ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:23.535334381Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83\") failed: Property not found. retry in 801.726727ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:23.543008622Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d\") failed: Property not found. retry in 597.110841ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:23.55054418Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069\") failed: Property not found. retry in 835.907411ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:23.558137059Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451\") failed: Property not found. retry in 1.199775322s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:23.565599906Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\") failed: Property not found. retry in 774.187403ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:23.585210931Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf\") failed: Property not found. retry in 1.06990322s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:23.608531249Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2\") failed: Property not found. retry in 1.069432251s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:23.679666022Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8\") failed: Property not found. retry in 1.262450996s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:23.882662214Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8\") failed: Property not found. retry in 1.384170881s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:24.059419238Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631\") failed: Property not found. retry in 1.480893566s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:24.09587349Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d\") failed: Property not found. retry in 550.787288ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:24.15688035Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65\") failed: Property not found. retry in 750.396955ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:24.350287229Z","hostname":"buskin","pid":1555,"zone":"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83\") failed: Property not found. retry in 960.565465ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:24.358436318Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\") failed: Property not found. retry in 587.720129ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:24.367868379Z","hostname":"buskin","pid":1555,"zone":"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d\") failed: Property not found. retry in 829.968088ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:24.719062337Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2\") failed: Property not found. retry in 663.079445ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:24.760930471Z","hostname":"buskin","pid":1555,"zone":"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2","component":"ServiceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"failed to register as a producer with Nexus, will retry","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:25.345792305Z","hostname":"buskin","pid":1555,"component":"producer-registration-task","file":"oximeter/producer/src/lib.rs:424","component":"producer-server","component":"MetricsManager","sled_id":"3abf3a4f-8820-40af-a01f-4369b42a5529","component":"SledAgent","file":"oximeter/producer/src/lib.rs:424","error":"\"Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/metrics/producers): error trying to connect: tcp connect error: Connection refused (os error 146)\"","delay":"62.221185713s"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:25.615276651Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3061","zone":"oxz_nexus_1c82ddd6-6f5a-4a92-a9b3-fde01c046c42"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:25.615362322Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3061","zone":"oxz_oximeter_893afdd2-39c6-4f94-b381-facb1a40bbb1"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:25.615375162Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3061","zone":"oxz_crucible_pantry_c02f008f-19c7-48c5-93c6-3f776c51e015"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:25.615381653Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3061","zone":"oxz_external_dns_8995285b-5cb0-4d3a-b021-8aec05314a65"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:25.615386182Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3061","zone":"oxz_external_dns_8a344ff3-4023-4f5e-a29a-3db2dc6980b8"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:25.615395173Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3061","zone":"oxz_clickhouse_061f99f8-6626-4a81-8262-abd77ceeb36a"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:25.615401683Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3061","zone":"oxz_crucible_pantry_738351b7-dacb-486a-ab38-cf8aefaef9cb"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:25.615411173Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3061","zone":"oxz_crucible_29db7ccb-90e1-414b-aeee-a9393f639631"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:25.615416373Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3061","zone":"oxz_crucible_8029adce-2a1d-48e9-a435-7bcff4f3bb83"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:25.615422363Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3061","zone":"oxz_crucible_48f67680-6ea6-4e0c-9968-c45baf393c1d"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:25.615428663Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3061","zone":"oxz_crucible_58b140e5-f61e-4410-8851-af929dfd8069"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:25.615433273Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3061","zone":"oxz_crucible_f5fbccc2-b8b1-4dd2-9655-42d226ad2451"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:25.615441854Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3061","zone":"oxz_crucible_19f0f364-8840-4b44-9eea-e3a6f21b6300"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:25.615463544Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3061","zone":"oxz_crucible_c1741444-7ab7-4888-aa3c-e7b6da3527da"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:25.615468914Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3061","zone":"oxz_nexus_866d6328-4ca1-435e-8a98-cd5268df29e8"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:25.615474394Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3061","zone":"oxz_crucible_d857801a-a95b-4937-a2f0-31ad7c239ac7"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:25.615478954Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3061","zone":"oxz_crucible_0d925b8a-ac40-4a91-9334-d6f7d7a29fb2"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:25.615582296Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3061","zone":"oxz_nexus_4952ec5d-23ba-48ea-9509-011fcb02adaf"} {"msg":"Zone started","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:25.615590856Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3061","zone":"oxz_crucible_pantry_15711896-d3e4-44e3-b330-624f0f5e064d"} {"msg":"Writing ledger to /pool/int/4576fedf-479b-43b3-8fad-bb8b7ec274f8/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:25.617773953Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/d09b3f64-0a36-4052-8233-a94e62e662e2/config/.omicron-zones.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:25.64222956Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:199"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:25.657862077Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"PUT","req_id":"5a24faad-e026-416a-bfc3-acaab4c101c5","remote_addr":"[fd00:1122:3344:101::1]:59148","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":36653109,"response_code":"204"} {"msg":"Finished setting up services","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:25.692365815Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:1250"} {"msg":"Writing ledger to /pool/int/4576fedf-479b-43b3-8fad-bb8b7ec274f8/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:25.69265661Z","hostname":"buskin","pid":1555,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/d09b3f64-0a36-4052-8233-a94e62e662e2/config/.rss-plan-completed.marker.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:25.692894114Z","hostname":"buskin","pid":1555,"component":"RSS","file":"common/src/ledger.rs:199"} {"msg":"Handing off control to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:25.705474809Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:687"} {"msg":"Nexus address: [fd00:1122:3344:101::c]:12221","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:25.71258932Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:700"} {"msg":"rack_network_config: RackNetworkConfigV2 {\n bfd: [],\n bgp: [],\n infra_ip_first: 10.151.1.174,\n infra_ip_last: 10.151.1.174,\n ports: [\n PortConfigV2 {\n addresses: [\n UplinkAddressConfig {\n address: V4(\n Ipv4Net {\n addr: 10.151.1.174,\n width: 24,\n },\n ),\n vlan_id: None,\n },\n ],\n autoneg: false,\n bgp_peers: [],\n port: \"qsfp0\",\n routes: [\n RouteConfig {\n destination: V4(\n Ipv4Net {\n addr: 0.0.0.0,\n width: 0,\n },\n ),\n nexthop: 10.151.1.100,\n vlan_id: None,\n },\n ],\n switch: Switch0,\n uplink_port_fec: None,\n uplink_port_speed: Speed40G,\n },\n ],\n rack_subnet: Ipv6Net {\n addr: fd00:1122:3344:100::,\n width: 56,\n },\n}","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:25.786970629Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:831"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/b682b2a4-3c99-42b2-9e14-9bba53281f21/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:25.826700086Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:893"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/b682b2a4-3c99-42b2-9e14-9bba53281f21/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:25.914240299Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:893"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/b682b2a4-3c99-42b2-9e14-9bba53281f21/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:26.074717646Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:893"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/b682b2a4-3c99-42b2-9e14-9bba53281f21/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:26.257206398Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:893"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/b682b2a4-3c99-42b2-9e14-9bba53281f21/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:26.434642844Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:893"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/b682b2a4-3c99-42b2-9e14-9bba53281f21/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:26.697074339Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:893"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/b682b2a4-3c99-42b2-9e14-9bba53281f21/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:26.891543775Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:893"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/b682b2a4-3c99-42b2-9e14-9bba53281f21/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:27.049891876Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:893"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/b682b2a4-3c99-42b2-9e14-9bba53281f21/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:27.328439646Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:893"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/b682b2a4-3c99-42b2-9e14-9bba53281f21/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:27.794428132Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:893"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/b682b2a4-3c99-42b2-9e14-9bba53281f21/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:28.54717459Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:893"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/b682b2a4-3c99-42b2-9e14-9bba53281f21/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:29.275775284Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:893"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/b682b2a4-3c99-42b2-9e14-9bba53281f21/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:29.878300109Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:893"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/3abf3a4f-8820-40af-a01f-4369b42a5529): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-08-05T16:46:30.940653616Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"3abf3a4f-8820-40af-a01f-4369b42a5529","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:30.963440284Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:56547"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:30.965755344Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"f0218010-ae46-4d07-bc51-2e3a6250e601","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:56547","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":739,"response_code":"200"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/b682b2a4-3c99-42b2-9e14-9bba53281f21/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:31.084963577Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:893"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/3abf3a4f-8820-40af-a01f-4369b42a5529): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:31.660598313Z","hostname":"buskin","pid":1555,"component":"NexusNotifierTask","sled_id":"3abf3a4f-8820-40af-a01f-4369b42a5529","component":"SledAgent","file":"sled-agent/src/nexus.rs:601"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/3abf3a4f-8820-40af-a01f-4369b42a5529): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-08-05T16:46:31.951734468Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"3abf3a4f-8820-40af-a01f-4369b42a5529","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/b682b2a4-3c99-42b2-9e14-9bba53281f21/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:32.485312828Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:893"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/3abf3a4f-8820-40af-a01f-4369b42a5529): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-08-05T16:46:32.95811374Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"3abf3a4f-8820-40af-a01f-4369b42a5529","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/3abf3a4f-8820-40af-a01f-4369b42a5529): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:33.24425647Z","hostname":"buskin","pid":1555,"component":"NexusNotifierTask","sled_id":"3abf3a4f-8820-40af-a01f-4369b42a5529","component":"SledAgent","file":"sled-agent/src/nexus.rs:601"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/3abf3a4f-8820-40af-a01f-4369b42a5529): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-08-05T16:46:33.960931851Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"3abf3a4f-8820-40af-a01f-4369b42a5529","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/b682b2a4-3c99-42b2-9e14-9bba53281f21/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:33.986795722Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:893"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/3abf3a4f-8820-40af-a01f-4369b42a5529): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-08-05T16:46:34.969516781Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"3abf3a4f-8820-40af-a01f-4369b42a5529","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/3abf3a4f-8820-40af-a01f-4369b42a5529): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:35.23976912Z","hostname":"buskin","pid":1555,"component":"NexusNotifierTask","sled_id":"3abf3a4f-8820-40af-a01f-4369b42a5529","component":"SledAgent","file":"sled-agent/src/nexus.rs:601"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/3abf3a4f-8820-40af-a01f-4369b42a5529): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-08-05T16:46:35.972843501Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"3abf3a4f-8820-40af-a01f-4369b42a5529","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/b682b2a4-3c99-42b2-9e14-9bba53281f21/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:36.106319147Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:893"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/3abf3a4f-8820-40af-a01f-4369b42a5529): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-08-05T16:46:36.978391968Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"3abf3a4f-8820-40af-a01f-4369b42a5529","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/3abf3a4f-8820-40af-a01f-4369b42a5529): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:37.239658204Z","hostname":"buskin","pid":1555,"component":"NexusNotifierTask","sled_id":"3abf3a4f-8820-40af-a01f-4369b42a5529","component":"SledAgent","file":"sled-agent/src/nexus.rs:601"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/b682b2a4-3c99-42b2-9e14-9bba53281f21/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:37.669349751Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:893"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/3abf3a4f-8820-40af-a01f-4369b42a5529): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-08-05T16:46:37.981461864Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"3abf3a4f-8820-40af-a01f-4369b42a5529","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/3abf3a4f-8820-40af-a01f-4369b42a5529): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-08-05T16:46:38.984807314Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"3abf3a4f-8820-40af-a01f-4369b42a5529","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/3abf3a4f-8820-40af-a01f-4369b42a5529): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:39.240456634Z","hostname":"buskin","pid":1555,"component":"NexusNotifierTask","sled_id":"3abf3a4f-8820-40af-a01f-4369b42a5529","component":"SledAgent","file":"sled-agent/src/nexus.rs:601"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/b682b2a4-3c99-42b2-9e14-9bba53281f21/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:39.906457861Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:893"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/3abf3a4f-8820-40af-a01f-4369b42a5529): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-08-05T16:46:39.98732118Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"3abf3a4f-8820-40af-a01f-4369b42a5529","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/3abf3a4f-8820-40af-a01f-4369b42a5529): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-08-05T16:46:40.990925544Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"3abf3a4f-8820-40af-a01f-4369b42a5529","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/3abf3a4f-8820-40af-a01f-4369b42a5529): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:41.239585355Z","hostname":"buskin","pid":1555,"component":"NexusNotifierTask","sled_id":"3abf3a4f-8820-40af-a01f-4369b42a5529","component":"SledAgent","file":"sled-agent/src/nexus.rs:601"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/3abf3a4f-8820-40af-a01f-4369b42a5529): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-08-05T16:46:41.993108755Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"3abf3a4f-8820-40af-a01f-4369b42a5529","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"get target probe state: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/probes/3abf3a4f-8820-40af-a01f-4369b42a5529): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":50,"time":"2024-08-05T16:46:42.99500897Z","hostname":"buskin","pid":1555,"component":"ProbeManager","sled_id":"3abf3a4f-8820-40af-a01f-4369b42a5529","component":"SledAgent","file":"sled-agent/src/probe_manager.rs:225"} {"msg":"Received Error from Nexus for Get request: Communication Error: error sending request for url (http://_nexus._tcp.control-plane.oxide.internal:12221/sled-agents/3abf3a4f-8820-40af-a01f-4369b42a5529): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:46:43.240268553Z","hostname":"buskin","pid":1555,"component":"NexusNotifierTask","sled_id":"3abf3a4f-8820-40af-a01f-4369b42a5529","component":"SledAgent","file":"sled-agent/src/nexus.rs:601"} {"msg":"Failed to handoff to nexus: Communication Error: error sending request for url (http://[fd00:1122:3344:101::c]:12221/racks/b682b2a4-3c99-42b2-9e14-9bba53281f21/initialization-complete): error trying to connect: tcp connect error: Connection refused (os error 146)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:43.417862581Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:893"} {"msg":"Failed to handoff to nexus: Error Response: status: 404 Not Found; headers: {\"content-type\": \"application/json\", \"x-request-id\": \"14e941f4-8867-4455-8b04-08b8c23fa66f\", \"content-length\": \"173\", \"date\": \"Mon, 05 Aug 2024 16:46:46 GMT\"}; value: Error { error_code: Some(\"ObjectNotFound\"), message: \"not found: sled with id \\\"3abf3a4f-8820-40af-a01f-4369b42a5529\\\"\", request_id: \"14e941f4-8867-4455-8b04-08b8c23fa66f\" }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:46.509360691Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:893"} {"msg":"Successfully put SledAgentInfo to nexus","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:47.274518139Z","hostname":"buskin","pid":1555,"component":"NexusNotifierTask","sled_id":"3abf3a4f-8820-40af-a01f-4369b42a5529","component":"SledAgent","file":"sled-agent/src/nexus.rs:574","new_generation":"1","old_generation":"1"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:47.41002926Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:62785"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:47.412549963Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:47.504303258Z","hostname":"buskin","pid":1555,"uri":"/inventory","method":"GET","req_id":"2fbfae6b-15b2-47e1-8755-39b0e36698bd","remote_addr":"[fd00:1122:3344:101::c]:62785","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":93735,"response_code":"200"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:47.515294375Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Reading ledger from /pool/int/4576fedf-479b-43b3-8fad-bb8b7ec274f8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:47.515507829Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Reading ledger from /pool/int/d09b3f64-0a36-4052-8233-a94e62e662e2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:47.516597618Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:47.517087306Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"GET","req_id":"cd5802e3-b05b-4ac4-9a3c-b8545e124e9d","remote_addr":"[fd00:1122:3344:101::c]:62785","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":12338,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:49.547610843Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:54671"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:49.551877125Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:673","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0a07869d-c9ec-4ade-b114-97ceff54b60e, kind: Service { id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8 }, name: Name(\"external-dns-8a344ff3-4023-4f5e-a29a-3db2dc6980b8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 206, 204])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: c8555f5b-90c8-4b5b-b73e-5b0ba7ce0695, kind: Service { id: 8995285b-5cb0-4d3a-b021-8aec05314a65 }, name: Name(\"external-dns-8995285b-5cb0-4d3a-b021-8aec05314a65\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 233, 199])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 38978e97-2109-4173-8624-98de2b729958, kind: Service { id: 4952ec5d-23ba-48ea-9509-011fcb02adaf }, name: Name(\"nexus-4952ec5d-23ba-48ea-9509-011fcb02adaf\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 188])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 51730dfc-654f-49d5-a0b7-5f1b747fb3be, kind: Service { id: 1c82ddd6-6f5a-4a92-a9b3-fde01c046c42 }, name: Name(\"nexus-1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 28])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 6eedd1d7-7b63-4139-96c3-ad58e9233e7c, kind: Service { id: 866d6328-4ca1-435e-8a98-cd5268df29e8 }, name: Name(\"nexus-866d6328-4ca1-435e-8a98-cd5268df29e8\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 250])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:49.5522025Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:49.553057134Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:49.553071735Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:49.553094695Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:49.553102775Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:49.553120786Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:49.553146776Z","hostname":"buskin","pid":1555,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"0ef38b26-3e64-44ed-a802-90fa1ad1fb4b","remote_addr":"[fd00:1122:3344:101::c]:54671","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":5014,"response_code":"204"} {"msg":"Handoff to Nexus is complete","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:49.554383797Z","hostname":"buskin","pid":1555,"component":"RSS","file":"sled-agent/src/rack_setup/service.rs:903"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:49.852860367Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:33060"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:49.852946209Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:49.940026284Z","hostname":"buskin","pid":1555,"uri":"/inventory","method":"GET","req_id":"3ef1809a-a80f-422a-a500-9a386d396217","remote_addr":"[fd00:1122:3344:101::c]:33060","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":87041,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:49.940160396Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:57546"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:49.940243448Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Reading ledger from /pool/int/4576fedf-479b-43b3-8fad-bb8b7ec274f8/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:49.940314969Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:49.940704485Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:673","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0a07869d-c9ec-4ade-b114-97ceff54b60e, kind: Service { id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8 }, name: Name(\"external-dns-8a344ff3-4023-4f5e-a29a-3db2dc6980b8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 206, 204])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: c8555f5b-90c8-4b5b-b73e-5b0ba7ce0695, kind: Service { id: 8995285b-5cb0-4d3a-b021-8aec05314a65 }, name: Name(\"external-dns-8995285b-5cb0-4d3a-b021-8aec05314a65\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 233, 199])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 38978e97-2109-4173-8624-98de2b729958, kind: Service { id: 4952ec5d-23ba-48ea-9509-011fcb02adaf }, name: Name(\"nexus-4952ec5d-23ba-48ea-9509-011fcb02adaf\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 188])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 51730dfc-654f-49d5-a0b7-5f1b747fb3be, kind: Service { id: 1c82ddd6-6f5a-4a92-a9b3-fde01c046c42 }, name: Name(\"nexus-1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 28])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 6eedd1d7-7b63-4139-96c3-ad58e9233e7c, kind: Service { id: 866d6328-4ca1-435e-8a98-cd5268df29e8 }, name: Name(\"nexus-866d6328-4ca1-435e-8a98-cd5268df29e8\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 250])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Reading ledger from /pool/int/d09b3f64-0a36-4052-8233-a94e62e662e2/config/omicron-zones.json","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:49.940733106Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"common/src/ledger.rs:177"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:49.940780277Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:49.940845328Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:49.940867518Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:49.940887159Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:49.940914679Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:49.94095455Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:49.94098635Z","hostname":"buskin","pid":1555,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"6f642107-d751-49f0-a370-70a0863c6ccb","remote_addr":"[fd00:1122:3344:101::c]:57546","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":578,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:49.941056031Z","hostname":"buskin","pid":1555,"uri":"/omicron-zones","method":"GET","req_id":"fac316b5-7c4e-489b-acb2-8c5bbbb66a11","remote_addr":"[fd00:1122:3344:101::c]:33060","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":864,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:49.963765029Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:47565"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:49.964247277Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:673","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0a07869d-c9ec-4ade-b114-97ceff54b60e, kind: Service { id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8 }, name: Name(\"external-dns-8a344ff3-4023-4f5e-a29a-3db2dc6980b8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 206, 204])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: c8555f5b-90c8-4b5b-b73e-5b0ba7ce0695, kind: Service { id: 8995285b-5cb0-4d3a-b021-8aec05314a65 }, name: Name(\"external-dns-8995285b-5cb0-4d3a-b021-8aec05314a65\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 233, 199])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 38978e97-2109-4173-8624-98de2b729958, kind: Service { id: 4952ec5d-23ba-48ea-9509-011fcb02adaf }, name: Name(\"nexus-4952ec5d-23ba-48ea-9509-011fcb02adaf\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 188])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 51730dfc-654f-49d5-a0b7-5f1b747fb3be, kind: Service { id: 1c82ddd6-6f5a-4a92-a9b3-fde01c046c42 }, name: Name(\"nexus-1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 28])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 6eedd1d7-7b63-4139-96c3-ad58e9233e7c, kind: Service { id: 866d6328-4ca1-435e-8a98-cd5268df29e8 }, name: Name(\"nexus-866d6328-4ca1-435e-8a98-cd5268df29e8\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 250])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:49.964294658Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:49.964348149Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:49.964371139Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:49.964389779Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:49.96440717Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:49.96442564Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:49.96444052Z","hostname":"buskin","pid":1555,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"59b23a5f-3478-4887-bfd1-a5b6001025a3","remote_addr":"[fd00:1122:3344:101::a]:47565","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":390,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:49.987946361Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:37350"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:49.988194375Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:673","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0a07869d-c9ec-4ade-b114-97ceff54b60e, kind: Service { id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8 }, name: Name(\"external-dns-8a344ff3-4023-4f5e-a29a-3db2dc6980b8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 206, 204])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: c8555f5b-90c8-4b5b-b73e-5b0ba7ce0695, kind: Service { id: 8995285b-5cb0-4d3a-b021-8aec05314a65 }, name: Name(\"external-dns-8995285b-5cb0-4d3a-b021-8aec05314a65\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 233, 199])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 38978e97-2109-4173-8624-98de2b729958, kind: Service { id: 4952ec5d-23ba-48ea-9509-011fcb02adaf }, name: Name(\"nexus-4952ec5d-23ba-48ea-9509-011fcb02adaf\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 188])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 51730dfc-654f-49d5-a0b7-5f1b747fb3be, kind: Service { id: 1c82ddd6-6f5a-4a92-a9b3-fde01c046c42 }, name: Name(\"nexus-1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 28])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 6eedd1d7-7b63-4139-96c3-ad58e9233e7c, kind: Service { id: 866d6328-4ca1-435e-8a98-cd5268df29e8 }, name: Name(\"nexus-866d6328-4ca1-435e-8a98-cd5268df29e8\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 250])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:49.988239086Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:49.988270007Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:49.988293997Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:49.988303057Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:49.988336258Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:49.988366738Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:49.988373638Z","hostname":"buskin","pid":1555,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"0a65042b-735e-425f-be82-56ba8ec35f45","remote_addr":"[fd00:1122:3344:101::b]:37350","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":294,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:59.455822978Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:50984"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:59.456657432Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"90715199-5761-4bc9-80e3-3a4a257ba075","remote_addr":"[fd00:1122:3344:101::c]:50984","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":786,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:46:59.519700637Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"77ec3fce-079f-4919-902d-7ddc9f4584c8","remote_addr":"[fd00:1122:3344:101::c]:50984","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":4010,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:47:00.997895304Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:51624"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:47:00.997960775Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"76942a5b-db7e-435f-87fd-5191845a7b5e","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:51624","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":40,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:47:15.125523582Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:38501"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:47:15.125613184Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"7587186e-8bfd-4fe2-97b5-345f89fb3f56","remote_addr":"[fd00:1122:3344:101::c]:38501","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":79,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:47:15.133270005Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"ae082a3d-e6f2-41e7-b6f2-a3d8f3aa30c1","remote_addr":"[fd00:1122:3344:101::c]:38501","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":105,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:47:15.232647279Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:46236"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:47:15.232719381Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"6a872ec8-d41f-419e-9031-f599774f3c1a","remote_addr":"[fd00:1122:3344:101::b]:46236","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":49,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:47:15.241009152Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"3c61418e-33e4-45a2-ae9a-aaee7d5f563b","remote_addr":"[fd00:1122:3344:101::b]:46236","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":104,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:47:15.260980752Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:51151"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:47:15.261829457Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"fe46ba4e-9321-488b-bee8-40a9c30b5370","remote_addr":"[fd00:1122:3344:101::a]:51151","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":825,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:47:15.2696397Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"53f92e94-bdca-4c4b-9ded-0a0a27f817d1","remote_addr":"[fd00:1122:3344:101::a]:51151","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":68,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:47:15.269713241Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:50348"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:47:15.270117878Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"3c03ea6b-46dc-449d-9297-36f687ca0847","remote_addr":"[fd00:1122:3344:101::b]:50348","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":398,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:47:15.286933745Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:41855"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:47:15.287081148Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"f0dfbd1f-4565-4b76-8e40-2ed296fffd69","remote_addr":"[fd00:1122:3344:101::c]:41855","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":107,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:47:15.406553525Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:36320"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:47:15.407239007Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"1056bf2e-54e1-44ac-be91-eac10067dc5c","remote_addr":"[fd00:1122:3344:101::a]:36320","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":652,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:47:15.440950321Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:36293"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:47:15.443456214Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3823"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:47:15.505512633Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3834"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:47:15.532650565Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"707fe93c-d5ba-4280-aba4-47688d037222","remote_addr":"[fd00:1122:3344:101::b]:36293","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":91651,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:47:15.532807018Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:57167"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:47:15.53290812Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:48314"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:47:15.533008791Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3823"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:47:15.539192067Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"d861fabe-0d1c-4ddd-8045-a10a9a201fb1","remote_addr":"[fd00:1122:3344:101::b]:36293","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":6205,"response_code":"200"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:47:15.559446891Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3834"} {"msg":"Attempting to update network config with generation: 2, current_generation: 1","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:47:15.565515605Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:564"} {"msg":"Writing ledger to /pool/int/4576fedf-479b-43b3-8fad-bb8b7ec274f8/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:47:15.566292418Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Writing ledger to /pool/int/d09b3f64-0a36-4052-8233-a94e62e662e2/cluster/.bootstore-network-config.json.tmp","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:47:15.567282535Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"common/src/ledger.rs:199"} {"msg":"Broadcasting network config with generation 2","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:47:15.568225471Z","hostname":"buskin","pid":1555,"peer_id":"pc-buskin-i86pc","component":"bootstore","file":"bootstore/src/schemes/v0/peer.rs:648"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:47:15.568418324Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"PUT","req_id":"e436fdf4-c88e-465d-9a1c-b2c68308bd22","remote_addr":"[fd00:1122:3344:101::b]:36293","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":3386,"response_code":"204"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:47:15.582713298Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3823"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:47:15.582772679Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"e42ad4fe-b412-4b84-9d9a-608aa528f908","remote_addr":"[fd00:1122:3344:101::c]:57167","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":49813,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:47:15.586206218Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"a016a64b-4565-4c5d-b5cd-d72aeeb1fd8f","remote_addr":"[fd00:1122:3344:101::c]:57167","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":624,"response_code":"200"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:47:15.608703331Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3834"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:47:15.632713781Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"f2bdf0d1-edcb-46f1-9468-3884dbcd6f4c","remote_addr":"[fd00:1122:3344:101::a]:48314","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":99651,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:47:15.633153688Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"92f1415c-9067-45ca-9fea-f1e25b7c338e","remote_addr":"[fd00:1122:3344:101::a]:48314","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":120,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:47:31.030773873Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:42922"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:47:31.030834094Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"6273ba72-1864-40f0-ad87-f4416e51b464","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:42922","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":40,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:47:45.264917817Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:33138"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:47:45.265138531Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"7c2c44e7-7248-4006-bf43-66488c0b2355","remote_addr":"[fd00:1122:3344:101::a]:33138","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":197,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:47:45.290134037Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:33120"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:47:45.290231139Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"8bec70a4-3506-4a60-8247-8c32872e6a18","remote_addr":"[fd00:1122:3344:101::b]:33120","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":49,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:47:45.299290453Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"e877faa7-485e-444e-89a2-913069364bd6","remote_addr":"[fd00:1122:3344:101::b]:33120","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":111,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:47:45.331652725Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:64806"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:47:45.332255675Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"f26fad78-d6ec-4ab4-89d7-94543b4fe668","remote_addr":"[fd00:1122:3344:101::c]:64806","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":581,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:47:45.341765008Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"d353de38-760a-4c6c-84e1-ecd64ebfa5a5","remote_addr":"[fd00:1122:3344:101::c]:64806","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":1051,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:47:45.44218073Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:60844"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:47:45.442307292Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"cf58d1ea-ecf8-437c-880c-583cb0ef9acc","remote_addr":"[fd00:1122:3344:101::a]:60844","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":91,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:47:45.450302779Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"6cacbb3c-733a-475d-8752-fe3f1a885156","remote_addr":"[fd00:1122:3344:101::a]:60844","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":116,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:47:45.481768865Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:40744"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:47:45.482936145Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"bf4a66a6-33ed-4157-9edf-3d958b605506","remote_addr":"[fd00:1122:3344:101::c]:40744","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":1082,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:47:45.506281063Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:44969"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:47:45.506391925Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"61d8640d-47b0-45b7-8c80-a957415cb111","remote_addr":"[fd00:1122:3344:101::b]:44969","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":76,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:47:45.616267308Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:44794"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:47:45.616341929Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3823"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:47:45.642253261Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3834"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:47:45.66385412Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"252b181f-d1b0-4778-ad94-ec53ec487bf8","remote_addr":"[fd00:1122:3344:101::c]:44794","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":47544,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:47:45.664053543Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:53546"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:47:45.664112474Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:38182"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:47:45.664749835Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3823"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:47:45.667843438Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"423694f3-bf43-49e4-88ac-b9ea5acbd228","remote_addr":"[fd00:1122:3344:101::c]:44794","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":3599,"response_code":"200"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:47:45.689520847Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3834"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:47:45.711882799Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3823"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:47:45.71196198Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"948ce42d-48f6-4e24-9759-b3ce5ebd6259","remote_addr":"[fd00:1122:3344:101::a]:53546","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":47810,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:47:45.715578502Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"394f62a1-12ba-477b-bb16-b2a3dc05e92a","remote_addr":"[fd00:1122:3344:101::a]:53546","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":649,"response_code":"200"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:47:45.738390571Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3834"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:47:45.759897087Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"9d8c7e5c-35e6-405a-9919-309c484340ba","remote_addr":"[fd00:1122:3344:101::b]:38182","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":95673,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:47:45.760354965Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"dc8ed8e8-9d1f-4af8-b69d-55cf03449bd4","remote_addr":"[fd00:1122:3344:101::b]:38182","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":182,"response_code":"200"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:47:47.246160193Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:47:57.701245362Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:40509","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","component":"producer-server","component":"MetricsManager","sled_id":"3abf3a4f-8820-40af-a01f-4369b42a5529","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::d]:56611"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:47:57.710385508Z","hostname":"buskin","pid":1555,"uri":"/3abf3a4f-8820-40af-a01f-4369b42a5529","method":"GET","req_id":"76dbeeca-6037-4445-8b18-86638edb1a9a","remote_addr":"[fd00:1122:3344:101::d]:56611","local_addr":"[fd00:1122:3344:101::1]:40509","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","component":"producer-server","component":"MetricsManager","sled_id":"3abf3a4f-8820-40af-a01f-4369b42a5529","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":8629,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:48:01.063687211Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:39187"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:48:01.064975133Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"cfbc08a1-c645-4821-b2c0-51132197bc5e","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:39187","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":589,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:48:15.091622408Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:50925"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:48:15.092786817Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"51327236-6626-4c33-b1c3-040dbb48f874","remote_addr":"[fd00:1122:3344:101::b]:50925","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":1150,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:48:15.100249505Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"c1717e0f-e37c-42fa-9437-5a07bba7488c","remote_addr":"[fd00:1122:3344:101::b]:50925","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":140,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:48:15.145200991Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:64742"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:48:15.145269103Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"f449888c-922f-4173-96b8-d477492514a4","remote_addr":"[fd00:1122:3344:101::c]:64742","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":43,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:48:15.153698646Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"f5183f4a-015b-4409-993a-1962dafd376d","remote_addr":"[fd00:1122:3344:101::c]:64742","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":86,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:48:15.201894868Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:43812"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:48:15.202723382Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"519295af-5b23-4848-baf0-4d26062f5547","remote_addr":"[fd00:1122:3344:101::a]:43812","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":789,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:48:15.210335792Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"b3607089-3f25-4009-ad69-3f9ae428e0cf","remote_addr":"[fd00:1122:3344:101::a]:43812","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":75,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:48:15.21487157Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:54818"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:48:15.215035052Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"2fa92a68-5f45-4f02-95bc-24a8004ddece","remote_addr":"[fd00:1122:3344:101::b]:54818","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":133,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:48:15.23481111Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:41021"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:48:15.234941972Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"3635fdc5-64e5-4a50-98cb-389611d1cc69","remote_addr":"[fd00:1122:3344:101::c]:41021","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":104,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:48:15.279857018Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:43991"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:48:15.279939679Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3823"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:48:15.30517283Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3834"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:48:15.326714777Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"335a7ab2-af6e-48a2-b38f-61ff174968e0","remote_addr":"[fd00:1122:3344:101::b]:43991","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":46803,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:48:15.32687118Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:45915"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:48:15.327153834Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"f891c9a5-b4cf-4599-9158-0c61a4537239","remote_addr":"[fd00:1122:3344:101::a]:45915","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":204,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:48:15.327924167Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"4eca3c1b-71dd-4dfe-b8ce-d89ccccf53e8","remote_addr":"[fd00:1122:3344:101::b]:43991","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":887,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:48:15.329887051Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:40400"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:48:15.330020953Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3823"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:48:15.351891236Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3834"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:48:15.371765025Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"c5e702f8-17c6-47ab-be81-fb323af6e4e2","remote_addr":"[fd00:1122:3344:101::c]:40400","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":41793,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:48:15.371965909Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:54293"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:48:15.372141872Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3823"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:48:15.374948099Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"feb32142-fdf6-4eda-817f-bb565b5826e9","remote_addr":"[fd00:1122:3344:101::c]:40400","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":2811,"response_code":"200"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:48:15.393258172Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3834"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:48:15.4148316Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"5ab536f8-9511-49e4-9b6d-77260a73029b","remote_addr":"[fd00:1122:3344:101::a]:54293","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":42775,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:48:15.415152205Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"417441c1-aab0-49af-a26c-3f606ad6a59f","remote_addr":"[fd00:1122:3344:101::a]:54293","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":129,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:48:27.701325769Z","hostname":"buskin","pid":1555,"uri":"/3abf3a4f-8820-40af-a01f-4369b42a5529","method":"GET","req_id":"70b35226-a9f9-4835-a916-a251f3f388f1","remote_addr":"[fd00:1122:3344:101::d]:56611","local_addr":"[fd00:1122:3344:101::1]:40509","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","component":"producer-server","component":"MetricsManager","sled_id":"3abf3a4f-8820-40af-a01f-4369b42a5529","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":1186,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:48:31.097228038Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:62362"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:48:31.09731067Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"a88d1c07-c2fe-460f-9039-26f273920bd0","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:62362","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":64,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:48:45.186500441Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:54451"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:48:45.186637454Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"f56c5ae7-2366-4d22-a4a1-872325ea38de","remote_addr":"[fd00:1122:3344:101::b]:54451","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":120,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:48:45.198263786Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"fd27d3b2-35f2-4945-ade9-5eff1bfc207c","remote_addr":"[fd00:1122:3344:101::b]:54451","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":103,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:48:45.253526145Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:58957"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:48:45.253601666Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"8b0e34de-cc78-4379-bfa8-cb51732ce565","remote_addr":"[fd00:1122:3344:101::c]:58957","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":35,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:48:45.260338711Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:53933"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:48:45.260412432Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"491e618d-121f-4044-84d8-406e0f0828a4","remote_addr":"[fd00:1122:3344:101::a]:53933","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":45,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:48:45.261508431Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:41091"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:48:45.263173299Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"d67d8530-058c-417b-897f-acff5214559d","remote_addr":"[fd00:1122:3344:101::b]:41091","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":1593,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:48:45.263192629Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"b8b84f23-8574-445a-9e6b-5bf67a572aec","remote_addr":"[fd00:1122:3344:101::c]:58957","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":1467,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:48:45.268092283Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"f5c1fbc9-d402-46e0-ade2-08a04bc47737","remote_addr":"[fd00:1122:3344:101::a]:53933","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":151,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:48:45.276969984Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:63956"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:48:45.277122787Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"b5536c2a-19bc-4c2e-a659-343ad13d8539","remote_addr":"[fd00:1122:3344:101::c]:63956","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":122,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:48:45.341404383Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:60885"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:48:45.341546496Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"1af94936-b2cb-48bb-9575-c071ad0e1466","remote_addr":"[fd00:1122:3344:101::a]:60885","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":114,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:48:45.51127981Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:34560"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:48:45.511585256Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3823"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:48:45.536524741Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3834"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:48:45.560142643Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"86ee9f10-d2e7-495e-8d68-0734cd1a8c80","remote_addr":"[fd00:1122:3344:101::c]:34560","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":48805,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:48:45.560265435Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:60507"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:48:45.560418837Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3823"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:48:45.563773495Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"221ff496-afab-4d5e-a26f-a97502a8c14f","remote_addr":"[fd00:1122:3344:101::c]:34560","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":3289,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:48:45.583644153Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:32844"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:48:45.585097618Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3834"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:48:45.606094487Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3823"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:48:45.606176718Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"24a21b38-cf83-4454-9ad7-78f0e2434856","remote_addr":"[fd00:1122:3344:101::a]:60507","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":45807,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:48:45.609551215Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"531bbb60-71be-43be-b1ed-59d37b465622","remote_addr":"[fd00:1122:3344:101::a]:60507","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":405,"response_code":"200"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:48:45.627745096Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3834"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:48:45.648443439Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"3518ad88-809c-4ad7-a7e3-a0dc1b2ae66c","remote_addr":"[fd00:1122:3344:101::b]:32844","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":64707,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:48:45.648754044Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"df5171a2-20cf-4050-8e3c-c0a8d3c9a800","remote_addr":"[fd00:1122:3344:101::b]:32844","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":124,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:48:57.701654071Z","hostname":"buskin","pid":1555,"uri":"/3abf3a4f-8820-40af-a01f-4369b42a5529","method":"GET","req_id":"7a2d76c3-e234-4f00-8142-85314768d3c8","remote_addr":"[fd00:1122:3344:101::d]:56611","local_addr":"[fd00:1122:3344:101::1]:40509","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","component":"producer-server","component":"MetricsManager","sled_id":"3abf3a4f-8820-40af-a01f-4369b42a5529","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":837,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:49:01.129522366Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:41664"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:49:01.129583407Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"2d5a930d-b8ed-4d17-9da8-34295d6728aa","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:41664","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":34,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:49:15.076271668Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:51198"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:49:15.07639344Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"eed75bf8-f747-4adf-ac1c-36d3e957bbe1","remote_addr":"[fd00:1122:3344:101::b]:51198","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":92,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:49:15.085939072Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"206528f2-eec2-4c35-8bf4-b7d56b218853","remote_addr":"[fd00:1122:3344:101::b]:51198","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":1187,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:49:15.133535204Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:61754"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:49:15.133613646Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"f37ff7d7-6d91-4467-85b2-fbb745561b75","remote_addr":"[fd00:1122:3344:101::a]:61754","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":26,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:49:15.141663103Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"597efe00-be9c-4e7f-8ffe-f3197b4484eb","remote_addr":"[fd00:1122:3344:101::a]:61754","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":1341,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:49:15.198170066Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:53168"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:49:15.198308389Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"5a6b52b5-0125-49f5-8578-32ba3ecd909d","remote_addr":"[fd00:1122:3344:101::c]:53168","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":89,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:49:15.205714795Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"5e00213a-d1d8-4e68-a9a4-c715b7306c65","remote_addr":"[fd00:1122:3344:101::c]:53168","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":1147,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:49:15.207128499Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:36407"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:49:15.207295622Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"de500518-8efb-4cbc-8364-2ba6fff00651","remote_addr":"[fd00:1122:3344:101::a]:36407","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":135,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:49:15.235660886Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:40685"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:49:15.235804048Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"c561a37b-4a10-4ab7-ba50-dab6fbf70e3f","remote_addr":"[fd00:1122:3344:101::b]:40685","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":89,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:49:15.309939543Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:36240"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:49:15.310016684Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3823"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:49:15.334167956Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3834"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:49:15.358084094Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"5f687014-3395-40dc-9f6c-7eb732b60d41","remote_addr":"[fd00:1122:3344:101::b]:36240","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":48101,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:49:15.358208776Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:59690"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:49:15.35842613Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"cd404043-a3ad-4ed4-8fad-e6db06b0dcc1","remote_addr":"[fd00:1122:3344:101::c]:59690","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":169,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:49:15.358548112Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"3c632655-22f1-4425-9b5f-51a0d86f362a","remote_addr":"[fd00:1122:3344:101::b]:36240","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":208,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:49:15.385940339Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:46054"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:49:15.386098091Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3823"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:49:15.409330658Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3834"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:49:15.431076828Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"92f47cb4-dd5b-4333-9bb3-51cf9d0c64d7","remote_addr":"[fd00:1122:3344:101::c]:46054","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":45040,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:49:15.431343983Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"f99d436e-5596-4be0-a64a-8f2df0b5efbe","remote_addr":"[fd00:1122:3344:101::c]:46054","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":95,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:49:15.432081235Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:53165"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:49:15.432230648Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3823"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:49:15.456224267Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3834"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:49:15.477434869Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"e3c73e91-8eea-4492-b8c0-4457fa18c043","remote_addr":"[fd00:1122:3344:101::a]:53165","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":45261,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:49:15.478171981Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"516ce4dd-039a-4311-ac4c-218391d78752","remote_addr":"[fd00:1122:3344:101::a]:53165","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":572,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:49:27.701579215Z","hostname":"buskin","pid":1555,"uri":"/3abf3a4f-8820-40af-a01f-4369b42a5529","method":"GET","req_id":"2df2b0db-3fdb-4b9b-a958-331c9ac29015","remote_addr":"[fd00:1122:3344:101::d]:56611","local_addr":"[fd00:1122:3344:101::1]:40509","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","component":"producer-server","component":"MetricsManager","sled_id":"3abf3a4f-8820-40af-a01f-4369b42a5529","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":647,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:49:31.161811592Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:53390"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:49:31.161875113Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"f858f876-6346-452e-96e6-404914cd9eb9","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:53390","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":39,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:49:45.206929321Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:42427"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:49:45.207139894Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"71ed3735-b4c7-47cf-aed9-7bca4ef33988","remote_addr":"[fd00:1122:3344:101::b]:42427","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":188,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:49:45.271613094Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:43569"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:49:45.271722205Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"c79ed037-c9c2-4383-9e1f-3c19446a2d63","remote_addr":"[fd00:1122:3344:101::a]:43569","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":61,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:49:45.280864532Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"d6065a67-4dbd-4133-9bcf-7660b19f41d3","remote_addr":"[fd00:1122:3344:101::a]:43569","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":133,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:49:45.361960944Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:32811"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:49:45.362054876Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"ced2f485-cf6f-497e-8b9f-5b7a87ad7819","remote_addr":"[fd00:1122:3344:101::b]:32811","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":69,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:49:45.369824129Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"86978eb3-c4c2-4a62-af96-537a8f51e6d8","remote_addr":"[fd00:1122:3344:101::b]:32811","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":103,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:49:45.419013237Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:39414"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:49:45.41918553Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"90791c85-8f34-4a13-9f47-1be7c44952ff","remote_addr":"[fd00:1122:3344:101::a]:39414","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":137,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:49:45.422797392Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:44367"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:49:45.422881463Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"0043024f-b41a-49dc-8bee-a9d0238c7337","remote_addr":"[fd00:1122:3344:101::c]:44367","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":34,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:49:45.430133037Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"94b55c49-939c-4659-bf10-52661b2f70e1","remote_addr":"[fd00:1122:3344:101::c]:44367","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":99,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:49:45.550256215Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:51268"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:49:45.550380517Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"4758d9c9-3fb6-4f9c-89d9-68772fc27c4b","remote_addr":"[fd00:1122:3344:101::c]:51268","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":101,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:49:45.560801195Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:56603"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:49:45.561553028Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3823"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:49:45.585897883Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3834"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:49:45.608894115Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"08ed2e69-9b60-45e8-adf1-0039bf560f59","remote_addr":"[fd00:1122:3344:101::a]:56603","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":48035,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:49:45.609989404Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"56c9feeb-caee-4e51-b72a-bd9ac9811f02","remote_addr":"[fd00:1122:3344:101::a]:56603","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":940,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:49:45.626719719Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:60632"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:49:45.626869261Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3823"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:49:45.650911491Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3834"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:49:45.67018849Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"20e15a2e-d558-4f4e-b9e8-70e7f4eeaa87","remote_addr":"[fd00:1122:3344:101::b]:60632","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":43390,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:49:45.670457865Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"fe335a10-81be-4969-937a-78af8aab0e6c","remote_addr":"[fd00:1122:3344:101::b]:60632","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":95,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:49:45.685303618Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:56495"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:49:45.685461841Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3823"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:49:45.709717905Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3834"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:49:45.734419846Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"79c2795a-a0fa-4792-82fb-f9ad23e5f048","remote_addr":"[fd00:1122:3344:101::c]:56495","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":49019,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:49:45.73468327Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"f2ef4bab-2495-44ba-8c87-80f077c85681","remote_addr":"[fd00:1122:3344:101::c]:56495","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":106,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:49:57.723428872Z","hostname":"buskin","pid":1555,"uri":"/3abf3a4f-8820-40af-a01f-4369b42a5529","method":"GET","req_id":"6b76b066-1f35-4798-ab97-be06703e7e7a","remote_addr":"[fd00:1122:3344:101::d]:56611","local_addr":"[fd00:1122:3344:101::1]:40509","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","component":"producer-server","component":"MetricsManager","sled_id":"3abf3a4f-8820-40af-a01f-4369b42a5529","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":357,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:50:01.194632136Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:62492"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:50:01.194777689Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"c52cd505-74a9-4602-bd8c-3a0154c6e629","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:62492","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":108,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:50:15.096113646Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:59447"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:50:15.096223537Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"2cc95b78-63af-4bb9-8cc3-877dc9341a55","remote_addr":"[fd00:1122:3344:101::b]:59447","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":94,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:50:15.103550742Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"78008d40-6051-404b-8bbc-0b70d51a1119","remote_addr":"[fd00:1122:3344:101::b]:59447","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":133,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:50:15.108946374Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:46299"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:50:15.109117647Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"5b67027f-6b59-48dd-ba59-9ea252d4cb1c","remote_addr":"[fd00:1122:3344:101::b]:46299","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":127,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:50:15.125938524Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:35942"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:50:15.126012125Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"f9f2ae37-c565-4927-907c-c799ccf36276","remote_addr":"[fd00:1122:3344:101::a]:35942","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":35,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:50:15.134362408Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"fed3e4db-24fd-41c9-9e9e-ae8206e8423f","remote_addr":"[fd00:1122:3344:101::a]:35942","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":57,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:50:15.144412199Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:58188"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:50:15.145391236Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"7b930c13-4099-497e-b82c-4c55180be386","remote_addr":"[fd00:1122:3344:101::c]:58188","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":921,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:50:15.153146438Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"d3aeb3f2-5c04-4f17-82a1-5af3018fe51e","remote_addr":"[fd00:1122:3344:101::c]:58188","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":48,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:50:15.153716378Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:37487"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:50:15.153816519Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"e2844057-7da9-4a33-a799-96a6d687dc82","remote_addr":"[fd00:1122:3344:101::c]:37487","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":77,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:50:15.18667948Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:52238"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:50:15.186757931Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"351994a2-11bf-4688-a0d2-f798b0eb2b3e","remote_addr":"[fd00:1122:3344:101::a]:52238","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":60,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:50:15.260077442Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:39474"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:50:15.260162933Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3823"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:50:15.28636941Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3834"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:50:15.310005183Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"318b6098-61ca-4642-87aa-993ad7a07710","remote_addr":"[fd00:1122:3344:101::b]:39474","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":49882,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:50:15.310315648Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:44678"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:50:15.310341069Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:48544"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:50:15.310541382Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"a9957078-ceeb-4502-9dc9-29b1646720b7","remote_addr":"[fd00:1122:3344:101::b]:39474","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":285,"response_code":"200"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:50:15.310586893Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3823"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:50:15.334090094Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3834"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:50:15.352244323Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3823"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:50:15.352325095Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"7982c971-0909-443b-9ff1-8f6439e4a568","remote_addr":"[fd00:1122:3344:101::a]:44678","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":41863,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:50:15.355386817Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"28e49e5a-1a5d-42ba-93e8-9428c0fd8886","remote_addr":"[fd00:1122:3344:101::a]:44678","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":637,"response_code":"200"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:50:15.376779122Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3834"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:50:15.402541801Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"1f603a66-e915-4a68-a178-5353595a2792","remote_addr":"[fd00:1122:3344:101::c]:48544","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":92041,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:50:15.402893717Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"3afd9d29-5ea9-4199-a4c2-d1695484f1af","remote_addr":"[fd00:1122:3344:101::c]:48544","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":119,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:50:27.763071053Z","hostname":"buskin","pid":1555,"uri":"/3abf3a4f-8820-40af-a01f-4369b42a5529","method":"GET","req_id":"f4489420-fd5d-4672-9fd3-fcf0672f653a","remote_addr":"[fd00:1122:3344:101::d]:56611","local_addr":"[fd00:1122:3344:101::1]:40509","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","component":"producer-server","component":"MetricsManager","sled_id":"3abf3a4f-8820-40af-a01f-4369b42a5529","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":511,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:50:31.226485254Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:57624"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:50:31.226542285Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"4c854913-b569-4042-9292-10e249f1ecc5","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:57624","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":41,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:50:45.194208863Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:34487"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:50:45.194334675Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"1f59901e-1078-4221-8df2-ef64d9ec0853","remote_addr":"[fd00:1122:3344:101::b]:34487","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":110,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:50:45.203401769Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"3e9800f3-8c25-429b-ab5f-25e62842ac94","remote_addr":"[fd00:1122:3344:101::b]:34487","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":124,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:50:45.220223896Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:33950"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:50:45.221668401Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"855925a6-7f73-499c-aab8-e375dcb0c77c","remote_addr":"[fd00:1122:3344:101::a]:33950","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":1385,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:50:45.229300361Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:51602"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:50:45.229384852Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"c4047efb-0587-40d5-bc73-ff5b7cfed7e6","remote_addr":"[fd00:1122:3344:101::c]:51602","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":37,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:50:45.229509835Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"2ff7bec9-2ffb-4761-86c5-5e713e4f0150","remote_addr":"[fd00:1122:3344:101::a]:33950","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":81,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:50:45.237406649Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"9eff9528-a3e5-4db4-a8b0-c1dc72a82b45","remote_addr":"[fd00:1122:3344:101::c]:51602","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":50,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:50:45.358008836Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:41499"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:50:45.358174739Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"2b42f255-5679-4f73-8786-f4eb488b0433","remote_addr":"[fd00:1122:3344:101::c]:41499","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":141,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:50:45.405308053Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:35154"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:50:45.405437115Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"7f946572-98c7-4a68-8925-f11af4b22cf1","remote_addr":"[fd00:1122:3344:101::a]:35154","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":107,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:50:45.441237695Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:59405"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:50:45.441364438Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"e35b46d2-ce23-4b4c-a967-ae4b6c543953","remote_addr":"[fd00:1122:3344:101::b]:59405","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":91,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:50:45.574983316Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:36724"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:50:45.575091697Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3823"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:50:45.601598919Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3834"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:50:45.626717388Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"98fbf3c0-d404-40c8-9c9f-515e48f7f8d3","remote_addr":"[fd00:1122:3344:101::a]:36724","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":51674,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:50:45.62684635Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:53723"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:50:45.627019773Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3823"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:50:45.630662925Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"584dd7b0-dbf1-4a9a-9b67-f2ec18fdab7e","remote_addr":"[fd00:1122:3344:101::a]:36724","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":3578,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:50:45.633159598Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:50157"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:50:45.650474843Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3834"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:50:45.672151222Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3823"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:50:45.672230594Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"3edd9244-3b05-43f9-a6e2-19e600033f63","remote_addr":"[fd00:1122:3344:101::c]:53723","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":45280,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:50:45.675752194Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"c8622fc1-7fe0-4c40-b674-1b7c84bafeb1","remote_addr":"[fd00:1122:3344:101::c]:53723","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":692,"response_code":"200"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:50:45.694647186Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3834"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:50:45.71655831Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"3b3bc697-15cc-43e3-b6f3-8cf430ee8065","remote_addr":"[fd00:1122:3344:101::b]:50157","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":83315,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:50:45.716946937Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"3b5c5aa4-9f49-4f45-a805-568e373ec0a3","remote_addr":"[fd00:1122:3344:101::b]:50157","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":160,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:50:57.763641357Z","hostname":"buskin","pid":1555,"uri":"/3abf3a4f-8820-40af-a01f-4369b42a5529","method":"GET","req_id":"c1117e33-d206-42f5-8900-65fbb7cc95f3","remote_addr":"[fd00:1122:3344:101::d]:56611","local_addr":"[fd00:1122:3344:101::1]:40509","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","component":"producer-server","component":"MetricsManager","sled_id":"3abf3a4f-8820-40af-a01f-4369b42a5529","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":575,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:01.258381182Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:39343"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:01.258478804Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"373e8a38-e69f-4d7a-b3b9-e2b8f26b656d","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:39343","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":46,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:15.112044946Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:43412"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:15.112966032Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"de32d076-ea06-457a-b684-c1e2915960f3","remote_addr":"[fd00:1122:3344:101::c]:43412","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":911,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:15.116475042Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:41687"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:15.116561203Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"64320c8a-2953-4a0e-8f9f-667fe00fd284","remote_addr":"[fd00:1122:3344:101::b]:41687","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":45,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:15.120013222Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"184bd984-fcb5-4182-ad75-1ce494de70f9","remote_addr":"[fd00:1122:3344:101::c]:43412","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":118,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:15.121453796Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:54240"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:15.121540098Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"67421fc7-2616-49ca-a3db-99f6e895fdb7","remote_addr":"[fd00:1122:3344:101::a]:54240","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":29,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:15.12400013Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"49b0ecb5-c946-4f56-8de7-7f4773001af5","remote_addr":"[fd00:1122:3344:101::b]:41687","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":882,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:15.12811643Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"21bc8d8d-06f4-42c4-a9bc-170b4f93ae57","remote_addr":"[fd00:1122:3344:101::a]:54240","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":47,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:15.169207641Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:44828"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:15.169356823Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"15a3a685-9302-4823-bb5d-8d65689038c3","remote_addr":"[fd00:1122:3344:101::a]:44828","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":128,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:15.169842852Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:47985"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:15.169950254Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"b2e0ba90-5862-4f14-8472-1d5e788577e1","remote_addr":"[fd00:1122:3344:101::c]:47985","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":68,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:15.22192347Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:38192"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:15.222872646Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"dc8e1665-9d50-4e60-b7c4-c4c12cb3db68","remote_addr":"[fd00:1122:3344:101::b]:38192","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":916,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:15.280898195Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:46123"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:15.280993557Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3823"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:15.304881704Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3834"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:15.327679663Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"ab711741-628a-4074-a7cd-577ac98ad178","remote_addr":"[fd00:1122:3344:101::a]:46123","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":46718,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:15.327823986Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:59438"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:15.327980628Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3823"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:15.33156133Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"adc333eb-18e8-4960-9f64-26ce23beb43d","remote_addr":"[fd00:1122:3344:101::a]:46123","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":3489,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:15.344250216Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:40647"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:15.352546387Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3834"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:15.374482502Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3823"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:15.374534122Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"be8627f5-2afe-4bce-906f-0c530e787652","remote_addr":"[fd00:1122:3344:101::b]:59438","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":46559,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:15.377976301Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"f08b9253-173b-4ce3-9a9a-cfe5ded4c0a8","remote_addr":"[fd00:1122:3344:101::b]:59438","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":996,"response_code":"200"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:15.396501277Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3834"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:15.41717243Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"08aeefae-5ebb-4ae3-9835-33baea7a9026","remote_addr":"[fd00:1122:3344:101::c]:40647","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":72757,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:15.417593337Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"a98ec130-b753-4c64-95ba-a5d01c3e39b8","remote_addr":"[fd00:1122:3344:101::c]:40647","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":151,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:27.764667209Z","hostname":"buskin","pid":1555,"uri":"/3abf3a4f-8820-40af-a01f-4369b42a5529","method":"GET","req_id":"0ee781b4-6510-4bc3-b114-c19ae0d32d20","remote_addr":"[fd00:1122:3344:101::d]:56611","local_addr":"[fd00:1122:3344:101::1]:40509","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","component":"producer-server","component":"MetricsManager","sled_id":"3abf3a4f-8820-40af-a01f-4369b42a5529","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":1180,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:31.291838848Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:59980"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:31.291904159Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"f1a3f3f8-d81f-4fba-ad8f-b1e3ae865a22","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:59980","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":47,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:45.146320096Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:52432"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:45.146443298Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"ea5ccb34-1522-461b-8d38-4d625e9f2ea8","remote_addr":"[fd00:1122:3344:101::b]:52432","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":105,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:45.153555219Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"7cecaa86-711d-42c7-b835-2ae07dc6ba85","remote_addr":"[fd00:1122:3344:101::b]:52432","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":123,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:45.199185778Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:47375"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:45.199436631Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"69e6d542-3cd3-4e4f-a346-419eadc71279","remote_addr":"[fd00:1122:3344:101::b]:47375","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":154,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:45.251603851Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:49483"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:45.251716223Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"09c888e6-7d2a-4a80-9c7e-545359d0f4a8","remote_addr":"[fd00:1122:3344:101::c]:49483","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":97,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:45.259532636Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"500dcdc8-7a80-44c2-9343-193e92db65d1","remote_addr":"[fd00:1122:3344:101::c]:49483","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":99,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:45.27852654Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:52293"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:45.278666702Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"f8189ec4-3e00-44cd-8fad-42aa153e3d2b","remote_addr":"[fd00:1122:3344:101::a]:52293","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":105,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:45.322027562Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:37999"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:45.322166584Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"997597ff-ad70-4ff3-8b54-c68543b28670","remote_addr":"[fd00:1122:3344:101::c]:37999","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":100,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:45.325023253Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:63735"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:45.325096024Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"4ee94b9c-be52-45df-92fb-4a7abc80e916","remote_addr":"[fd00:1122:3344:101::a]:63735","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":32,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:45.331833759Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"f5ec2b35-9e72-45ad-8fc6-46ed86cd5bb6","remote_addr":"[fd00:1122:3344:101::a]:63735","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":87,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:45.381290052Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:64464"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:45.381447105Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:673","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0a07869d-c9ec-4ade-b114-97ceff54b60e, kind: Service { id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8 }, name: Name(\"external-dns-8a344ff3-4023-4f5e-a29a-3db2dc6980b8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 206, 204])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: c8555f5b-90c8-4b5b-b73e-5b0ba7ce0695, kind: Service { id: 8995285b-5cb0-4d3a-b021-8aec05314a65 }, name: Name(\"external-dns-8995285b-5cb0-4d3a-b021-8aec05314a65\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 233, 199])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 38978e97-2109-4173-8624-98de2b729958, kind: Service { id: 4952ec5d-23ba-48ea-9509-011fcb02adaf }, name: Name(\"nexus-4952ec5d-23ba-48ea-9509-011fcb02adaf\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 188])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 51730dfc-654f-49d5-a0b7-5f1b747fb3be, kind: Service { id: 1c82ddd6-6f5a-4a92-a9b3-fde01c046c42 }, name: Name(\"nexus-1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 28])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 6eedd1d7-7b63-4139-96c3-ad58e9233e7c, kind: Service { id: 866d6328-4ca1-435e-8a98-cd5268df29e8 }, name: Name(\"nexus-866d6328-4ca1-435e-8a98-cd5268df29e8\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 250])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:45.381482636Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:45.381505696Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:45.381520116Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:45.381533537Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:45.381553207Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:45.381563977Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:45.381580348Z","hostname":"buskin","pid":1555,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"113e869f-3114-4578-9b74-6af2387f2e96","remote_addr":"[fd00:1122:3344:101::a]:64464","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":269,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:45.512725574Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:63585"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:45.512881017Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:673","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0a07869d-c9ec-4ade-b114-97ceff54b60e, kind: Service { id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8 }, name: Name(\"external-dns-8a344ff3-4023-4f5e-a29a-3db2dc6980b8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 206, 204])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: c8555f5b-90c8-4b5b-b73e-5b0ba7ce0695, kind: Service { id: 8995285b-5cb0-4d3a-b021-8aec05314a65 }, name: Name(\"external-dns-8995285b-5cb0-4d3a-b021-8aec05314a65\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 233, 199])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 38978e97-2109-4173-8624-98de2b729958, kind: Service { id: 4952ec5d-23ba-48ea-9509-011fcb02adaf }, name: Name(\"nexus-4952ec5d-23ba-48ea-9509-011fcb02adaf\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 188])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 51730dfc-654f-49d5-a0b7-5f1b747fb3be, kind: Service { id: 1c82ddd6-6f5a-4a92-a9b3-fde01c046c42 }, name: Name(\"nexus-1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 28])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 6eedd1d7-7b63-4139-96c3-ad58e9233e7c, kind: Service { id: 866d6328-4ca1-435e-8a98-cd5268df29e8 }, name: Name(\"nexus-866d6328-4ca1-435e-8a98-cd5268df29e8\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 250])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:45.512908367Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:45.512940658Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:45.512949818Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:45.512955468Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:45.512963348Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:45.512975738Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:45.512989298Z","hostname":"buskin","pid":1555,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"ade02128-5a44-4a1a-ad51-6d012a66ecb6","remote_addr":"[fd00:1122:3344:101::c]:63585","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":202,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:45.536606231Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:39198"} {"msg":"Ensuring VPC firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:45.536736634Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:673","rules":"[VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 0a07869d-c9ec-4ade-b114-97ceff54b60e, kind: Service { id: 8a344ff3-4023-4f5e-a29a-3db2dc6980b8 }, name: Name(\"external-dns-8a344ff3-4023-4f5e-a29a-3db2dc6980b8\"), ip: 172.30.1.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 206, 204])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: c8555f5b-90c8-4b5b-b73e-5b0ba7ce0695, kind: Service { id: 8995285b-5cb0-4d3a-b021-8aec05314a65 }, name: Name(\"external-dns-8995285b-5cb0-4d3a-b021-8aec05314a65\"), ip: 172.30.1.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 233, 199])), subnet: V4(Ipv4Net { addr: 172.30.1.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(53), last: L4Port(53) }]), filter_protocols: Some([Udp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 38978e97-2109-4173-8624-98de2b729958, kind: Service { id: 4952ec5d-23ba-48ea-9509-011fcb02adaf }, name: Name(\"nexus-4952ec5d-23ba-48ea-9509-011fcb02adaf\"), ip: 172.30.2.6, mac: MacAddr(MacAddr6([168, 64, 37, 255, 216, 188])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 51730dfc-654f-49d5-a0b7-5f1b747fb3be, kind: Service { id: 1c82ddd6-6f5a-4a92-a9b3-fde01c046c42 }, name: Name(\"nexus-1c82ddd6-6f5a-4a92-a9b3-fde01c046c42\"), ip: 172.30.2.7, mac: MacAddr(MacAddr6([168, 64, 37, 255, 202, 28])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }, NetworkInterface { id: 6eedd1d7-7b63-4139-96c3-ad58e9233e7c, kind: Service { id: 866d6328-4ca1-435e-8a98-cd5268df29e8 }, name: Name(\"nexus-866d6328-4ca1-435e-8a98-cd5268df29e8\"), ip: 172.30.2.5, mac: MacAddr(MacAddr6([168, 64, 37, 255, 210, 250])), subnet: V4(Ipv4Net { addr: 172.30.2.0, width: 24 }), vni: Vni(100), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(80), last: L4Port(80) }, L4PortRange { first: L4Port(443), last: L4Port(443) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }]","vni":"Vni(100)"} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:45.536753324Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte3\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:45.536772554Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte5\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:45.536786044Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte1\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:45.536794204Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[]","port":"\"opte0\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:45.536800164Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(TCP), ports: PortList([80, 443]) }, action: Allow, priority: 65534 }]","port":"\"opte4\""} {"msg":"Setting OPTE firewall rules","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:45.536807875Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:692","rules":"[FirewallRule { direction: In, filters: Filters { hosts: Any, protocol: Proto(UDP), ports: PortList([53]) }, action: Allow, priority: 65534 }]","port":"\"opte2\""} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:45.536820705Z","hostname":"buskin","pid":1555,"uri":"/vpc/001de000-074c-4000-8000-000000000000/firewall/rules","method":"PUT","req_id":"5c0694a6-44eb-4929-a37c-2156448f5f9b","remote_addr":"[fd00:1122:3344:101::b]:39198","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":178,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:45.574340484Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:54075"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:45.574418935Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3823"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:45.600011532Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3834"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:45.623188437Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"07b628d3-2fd4-49ea-9cf7-557419ca53a4","remote_addr":"[fd00:1122:3344:101::a]:54075","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":48803,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:45.623314199Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:43056"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:45.623476312Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3823"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:45.626566285Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"1a57c281-5ecc-4646-beb0-043fd413fc8b","remote_addr":"[fd00:1122:3344:101::a]:54075","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":3066,"response_code":"200"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:45.647678655Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3834"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:45.669791342Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"2c5bb108-48f9-4427-a70f-3e4332297eb7","remote_addr":"[fd00:1122:3344:101::c]:43056","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":46386,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:45.670130357Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"7c47232a-a8c2-4d0f-bfba-a4ffd02000bb","remote_addr":"[fd00:1122:3344:101::c]:43056","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":142,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:45.740880854Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:58619"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:45.740981936Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3823"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:45.762180967Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3834"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:45.784431987Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"d68103ec-eaa8-4560-8490-b495bfffab96","remote_addr":"[fd00:1122:3344:101::b]:58619","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":43487,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:45.785277241Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"31b13999-8944-4d8b-ad8a-5bbe696328fd","remote_addr":"[fd00:1122:3344:101::b]:58619","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":707,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:51:57.764891098Z","hostname":"buskin","pid":1555,"uri":"/3abf3a4f-8820-40af-a01f-4369b42a5529","method":"GET","req_id":"f2ed86a5-e24f-4abe-9b12-3235ad44741c","remote_addr":"[fd00:1122:3344:101::d]:56611","local_addr":"[fd00:1122:3344:101::1]:40509","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","component":"producer-server","component":"MetricsManager","sled_id":"3abf3a4f-8820-40af-a01f-4369b42a5529","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":1283,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:52:01.325078159Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:35291"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:52:01.32514672Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"ee1df76b-a09f-407c-92a8-44fbac2bd13c","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:35291","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":39,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:52:15.101594759Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:63927"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:52:15.101704281Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"33230b50-bc36-4d3b-b574-a1a9963fe276","remote_addr":"[fd00:1122:3344:101::a]:63927","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":96,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:52:15.110594462Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"9874bffc-efba-4ccc-b7e7-d5c70801e3ca","remote_addr":"[fd00:1122:3344:101::a]:63927","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":1083,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:52:15.110638233Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:39868"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:52:15.110693304Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"e913a43b-ab19-4f62-b1db-0477d808a5ee","remote_addr":"[fd00:1122:3344:101::b]:39868","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":31,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:52:15.117407259Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"759a84bf-1854-4a5a-91ef-740d5ef76573","remote_addr":"[fd00:1122:3344:101::b]:39868","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":75,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:52:15.132185411Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:43768"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:52:15.132348904Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"c46d4624-57e5-478f-8797-7eeaaca70913","remote_addr":"[fd00:1122:3344:101::a]:43768","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":128,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:52:15.152514997Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:36683"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:52:15.152615069Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"f15b3f8a-21cf-4efb-a0aa-add6d8e90c50","remote_addr":"[fd00:1122:3344:101::c]:36683","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":46,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:52:15.15972887Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"3862147f-b2b2-4828-8116-32e615a45235","remote_addr":"[fd00:1122:3344:101::c]:36683","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":72,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:52:15.192769204Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:35145"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:52:15.193639029Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"f4b2c02c-53ba-44d7-85c9-4721bdaed34b","remote_addr":"[fd00:1122:3344:101::c]:35145","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":845,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:52:15.200024527Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:64736"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:52:15.20015689Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"e66d72cb-a015-4a20-b413-f356a6ed8013","remote_addr":"[fd00:1122:3344:101::b]:64736","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":75,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:52:15.25527797Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:47446"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:52:15.256016993Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3823"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:52:15.280759214Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3834"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:52:15.304313236Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"3124e846-c835-4980-a7c7-92f622deea93","remote_addr":"[fd00:1122:3344:101::a]:47446","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":48978,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:52:15.304435908Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:35278"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:52:15.304625442Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3823"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:52:15.308174222Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"69fc20b8-ae6c-4438-ab7b-256b12cdeafa","remote_addr":"[fd00:1122:3344:101::a]:47446","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":3502,"response_code":"200"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:52:15.328717152Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3834"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:52:15.351177435Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"482ee68c-ed03-41bb-915f-f8827b6f74ae","remote_addr":"[fd00:1122:3344:101::b]:35278","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":46638,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:52:15.352138212Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"1138e5b8-2e37-4a47-be87-68d8aea0e730","remote_addr":"[fd00:1122:3344:101::b]:35278","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":734,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:52:15.373929123Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:40548"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:52:15.374032825Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3823"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:52:15.395975579Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3834"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:52:15.418040625Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"a37d1298-4b00-43f4-bbbd-c7d90bdf6001","remote_addr":"[fd00:1122:3344:101::c]:40548","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":44043,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:52:15.419181145Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"591b35b3-bc2b-4870-800e-c3d6ed8bb929","remote_addr":"[fd00:1122:3344:101::c]:40548","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":904,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:52:27.764175443Z","hostname":"buskin","pid":1555,"uri":"/3abf3a4f-8820-40af-a01f-4369b42a5529","method":"GET","req_id":"cfe2443e-cf37-4a2a-9a4f-dbf6a2be8f70","remote_addr":"[fd00:1122:3344:101::d]:56611","local_addr":"[fd00:1122:3344:101::1]:40509","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","component":"producer-server","component":"MetricsManager","sled_id":"3abf3a4f-8820-40af-a01f-4369b42a5529","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":537,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:52:31.359516424Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:39409"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:52:31.359622846Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"3b108dc9-72d8-4466-bc83-6c55ed6df27d","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:39409","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":86,"response_code":"200"} {"msg":"running automatic periodic zone bundle cleanup","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:52:33.455996416Z","hostname":"buskin","pid":1555,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:303"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:52:33.456481754Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"top of bundle cleanup loop","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:52:33.493107068Z","hostname":"buskin","pid":1555,"component":"auto-cleanup-task","component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:292","time_to_next_cleanup":"599.99999578s","next_cleanup":"Instant { tv_sec: 12090, tv_nsec: 850417773 }"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:52:45.186208909Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:45629"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:52:45.187902798Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"035ef1f2-7bae-49c3-9a79-8ceaad0caa06","remote_addr":"[fd00:1122:3344:101::a]:45629","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":1681,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:52:45.194605982Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"e24af0a8-2f68-4b7b-8ec3-0f4d16af220a","remote_addr":"[fd00:1122:3344:101::a]:45629","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":101,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:52:45.234012474Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:48904"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:52:45.234166597Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"cd1df7ca-a5e6-410e-8274-c3dcd934071c","remote_addr":"[fd00:1122:3344:101::a]:48904","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":130,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:52:45.360953519Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:39334"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:52:45.36100935Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"bf75c0c2-e5f7-49eb-9777-8b54d532fcde","remote_addr":"[fd00:1122:3344:101::b]:39334","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":24,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:52:45.368225513Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"92efe327-3be2-4492-94e6-6160ff1d39bf","remote_addr":"[fd00:1122:3344:101::b]:39334","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":78,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:52:45.373762998Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:47400"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:52:45.374776685Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"3dfbab9e-a5b9-4d7a-afd8-6ab7a72c6cbf","remote_addr":"[fd00:1122:3344:101::c]:47400","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":947,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:52:45.381600541Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"eee97850-da2d-490b-befa-6f19db620aa5","remote_addr":"[fd00:1122:3344:101::c]:47400","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":68,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:52:45.400032006Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:46676"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:52:45.400193028Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"e3de0d0a-5651-4402-b566-d963d49cd7bf","remote_addr":"[fd00:1122:3344:101::c]:46676","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":98,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:52:45.488851991Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:44772"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:52:45.489646824Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3823"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:52:45.512813569Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3834"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:52:45.535070629Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"9d6fa586-756f-4a02-806a-e9a3892ae54b","remote_addr":"[fd00:1122:3344:101::a]:44772","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":46166,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:52:45.535263402Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:40397"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:52:45.535285772Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:37731"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:52:45.535578687Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"7676728f-e7fe-4e52-b6b1-0607ad48734b","remote_addr":"[fd00:1122:3344:101::a]:44772","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":204,"response_code":"200"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:52:45.535666799Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3823"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:52:45.535781731Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"cd6bbcec-07b4-4020-8806-8b8b6b4e5330","remote_addr":"[fd00:1122:3344:101::b]:37731","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":304,"response_code":"200"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:52:45.558910505Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3834"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:52:45.578797464Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"92bb5965-7799-496a-9e6e-7e692e848e2f","remote_addr":"[fd00:1122:3344:101::c]:40397","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":43306,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:52:45.579828951Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"9b98ad89-3d8b-4eea-b58a-ed99b2db0e36","remote_addr":"[fd00:1122:3344:101::c]:40397","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":854,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:52:45.683460598Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:35560"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:52:45.68357008Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3823"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:52:45.705157759Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3834"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:52:45.726888089Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"b18f514a-9ce7-4794-abf4-d95ea7b9ea2f","remote_addr":"[fd00:1122:3344:101::b]:35560","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":43365,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:52:45.727137603Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"531e0cd4-8a30-4304-823d-4f8d06c2d481","remote_addr":"[fd00:1122:3344:101::b]:35560","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":92,"response_code":"200"} {"msg":"No archival destination for crash dumps yet chosen.","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:52:47.249342812Z","hostname":"buskin","pid":1555,"component":"DumpSetup-worker","file":"sled-agent/src/dump_setup.rs:862"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:52:57.841418267Z","hostname":"buskin","pid":1555,"uri":"/3abf3a4f-8820-40af-a01f-4369b42a5529","method":"GET","req_id":"4b6ee169-f9aa-47ad-b6be-c1b25ea78c80","remote_addr":"[fd00:1122:3344:101::d]:56611","local_addr":"[fd00:1122:3344:101::1]:40509","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","component":"producer-server","component":"MetricsManager","sled_id":"3abf3a4f-8820-40af-a01f-4369b42a5529","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":1431,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:01.392704434Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:55782"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:01.393416429Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"9f1a3c38-3675-4d01-8eec-0475ee26e7a6","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:55782","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":630,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:15.121800447Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:39366"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:15.121950929Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"5b266bdd-8cd6-42cf-bca5-7016c1518aca","remote_addr":"[fd00:1122:3344:101::b]:39366","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":99,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:15.12958517Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"0e56fdfa-5323-4ab8-8447-15b30cc70186","remote_addr":"[fd00:1122:3344:101::b]:39366","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":162,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:15.13313421Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:48109"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:15.133233652Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"fc55e361-4845-4a32-9a26-3f0612edc147","remote_addr":"[fd00:1122:3344:101::c]:48109","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":37,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:15.140274012Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:55103"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:15.140350363Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"13ff5be7-018a-41a5-9339-d00f765d6ff7","remote_addr":"[fd00:1122:3344:101::a]:55103","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":56,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:15.141470592Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"24c986d9-9a7f-45fe-a71f-b6403c297eb7","remote_addr":"[fd00:1122:3344:101::c]:48109","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":100,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:15.147373213Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"a4b88122-a0b8-4e6b-98a0-6a009235351e","remote_addr":"[fd00:1122:3344:101::a]:55103","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":74,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:15.155782284Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:52734"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:15.15599986Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"eb4d0b71-9269-4268-84a0-7d148142cd6c","remote_addr":"[fd00:1122:3344:101::a]:52734","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":140,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:15.174533806Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:35376"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:15.17474793Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"4acf2b60-913a-49bc-85a2-acf5c0716a98","remote_addr":"[fd00:1122:3344:101::c]:35376","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":130,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:15.215467274Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:37537"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:15.215647137Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"dc076068-5d2f-42ba-af9e-a645aa526403","remote_addr":"[fd00:1122:3344:101::b]:37537","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":142,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:15.287650615Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:53817"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:15.287747727Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3823"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:15.309507398Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3834"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:15.331529523Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"cefa6018-29f3-4b25-9c7d-6b5044215d60","remote_addr":"[fd00:1122:3344:101::a]:53817","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":43827,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:15.331678426Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::b]:61216"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:15.331850899Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3823"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:15.335067284Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"b14d4960-d286-46c3-8ae0-3dd48b41f2a1","remote_addr":"[fd00:1122:3344:101::a]:53817","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":3192,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:15.353847264Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:42554"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:15.35537147Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3834"} {"msg":"Setting up uplinkd service","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:15.376250646Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3823"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:15.376315177Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"978250ab-c604-475e-8aa1-c7fecab7b579","remote_addr":"[fd00:1122:3344:101::b]:61216","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":44527,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:15.379631224Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"4b4f838f-5bf6-47b2-991a-87eabf418e95","remote_addr":"[fd00:1122:3344:101::b]:61216","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":517,"response_code":"200"} {"msg":"configuring port: HostPortConfig { port: \"qsfp0\", addrs: [UplinkAddressConfig { address: V4(Ipv4Net { addr: 10.151.1.174, width: 24 }), vlan_id: None }] }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:15.400161834Z","hostname":"buskin","pid":1555,"component":"ServiceManager","file":"sled-agent/src/services.rs:3834"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:15.422887861Z","hostname":"buskin","pid":1555,"uri":"/switch-ports","method":"POST","req_id":"2bcf1f37-d715-42c8-b0f5-81aa1a1df064","remote_addr":"[fd00:1122:3344:101::c]:42554","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":68941,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:15.423259968Z","hostname":"buskin","pid":1555,"uri":"/network-bootstore-config","method":"GET","req_id":"40e83b8d-bcdd-48e3-b50d-9310d04d5972","remote_addr":"[fd00:1122:3344:101::c]:42554","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":126,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:18.940984984Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:63368"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:18.941784658Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"26dc6adb-6719-4fd2-a7a9-55f905cafa70","remote_addr":"[fd00:1122:3344:101::a]:63368","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":787,"response_code":"200"} {"msg":"Mapping virtual NIC to physical host","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:18.944942352Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:780","mapping":"VirtualNetworkInterfaceHost { virtual_ip: 172.30.0.5, virtual_mac: MacAddr(MacAddr6([168, 64, 37, 244, 103, 212])), physical_host_ip: fd00:1122:3344:101::1, vni: Vni(8395431) }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:18.945095695Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"PUT","req_id":"7de677ac-08ae-46a1-8168-d018b3261577","remote_addr":"[fd00:1122:3344:101::a]:63368","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":3146,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:19.066032507Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:63491"} {"msg":"ensuring instance is registered","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:19.073166248Z","hostname":"buskin","pid":1555,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:592","metadata":"InstanceMetadata { silo_id: e179a288-a562-4f81-9527-488ea538b84b, project_id: 19b9bd76-f1e4-448f-b46c-d56eb60fe928 }","propolis_addr":"[fd00:1122:3344:101::1:0]:12400","vmm_runtime":"VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-08-05T16:53:18.684161Z }","instance_runtime":"InstanceRuntimeState { propolis_id: Some(a99b853a-2530-488f-b74e-e3781d0d4d3c (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-08-05T16:53:18.731554843Z }","hardware":"InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: 7e30d6a5-06c5-48b0-a304-819497336a2b, kind: Instance { id: 60194fb6-a280-4030-a452-3f4a705fbb54 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 244, 103, 212])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(8395431), primary: true, slot: 0, transit_ips: [] }], source_nat: SourceNatConfig { ip: 10.151.1.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.1.176), floating_ips: [], firewall_rules: [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 7e30d6a5-06c5-48b0-a304-819497336a2b, kind: Instance { id: 60194fb6-a280-4030-a452-3f4a705fbb54 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 244, 103, 212])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(8395431), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 7e30d6a5-06c5-48b0-a304-819497336a2b, kind: Instance { id: 60194fb6-a280-4030-a452-3f4a705fbb54 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 244, 103, 212])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(8395431), primary: true, slot: 0, transit_ips: [] }], filter_hosts: Some([Vpc(Vni(8395431))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 7e30d6a5-06c5-48b0-a304-819497336a2b, kind: Instance { id: 60194fb6-a280-4030-a452-3f4a705fbb54 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 244, 103, 212])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(8395431), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-95063f50cc0\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 3b001efd-ef58-42b1-9232-aa581e636831, read_only_parent: Some(Volume { block_size: 512, id: 9f47f0e9-08aa-463b-8505-23feaed489ca, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 4845401b-1c93-4371-965c-ef447b2b4c4e, key: Some(\"wBLcPlGM1lU0a4cNONAjXCEYfJYa8e7+N/aJipdWaCw=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::12]:19001\", \"[fd00:1122:3344:101::18]:19001\", \"[fd00:1122:3344:101::14]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 3b001efd-ef58-42b1-9232-aa581e636831, key: Some(\"a4ZH8kEQG4WXhxpUzSemMPaL2KbplBadaQMt3A1Xox8=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::12]:19002\", \"[fd00:1122:3344:101::14]:19002\", \"[fd00:1122:3344:101::19]:19000\"] } }] } }], cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAABnqYYFWQVZAACphgVZAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAZ6mGBVkFWQAAqYYFWQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiI2MDE5NGZiNi1hMjgwLTQwMzAtYTQ1Mi0zZjRhNzA1ZmJiNTQiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJQlJ3VnF1TjY1Z0lCYWwwc1NXUzBaQ01ia3dVRlJxY0tlNGFVVW1DZGRYMSJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\") }","propolis_id":"a99b853a-2530-488f-b74e-e3781d0d4d3c","instance_id":"60194fb6-a280-4030-a452-3f4a705fbb54"} {"msg":"registering new instance","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:19.07326328Z","hostname":"buskin","pid":1555,"component":"InstanceManager","file":"sled-agent/src/instance_manager.rs:626","instance_id":"60194fb6-a280-4030-a452-3f4a705fbb54 (instance)"} {"msg":"initializing new Instance","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:19.073386052Z","hostname":"buskin","pid":1555,"component":"InstanceManager","file":"sled-agent/src/instance.rs:992","state":"InstanceInitialState { hardware: InstanceHardware { properties: InstanceProperties { ncpus: InstanceCpuCount(2), memory: ByteCount(1073741824), hostname: Hostname(\"localshark\") }, nics: [NetworkInterface { id: 7e30d6a5-06c5-48b0-a304-819497336a2b, kind: Instance { id: 60194fb6-a280-4030-a452-3f4a705fbb54 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 244, 103, 212])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(8395431), primary: true, slot: 0, transit_ips: [] }], source_nat: SourceNatConfig { ip: 10.151.1.175, first_port: 0, last_port: 16383 }, ephemeral_ip: Some(10.151.1.176), floating_ips: [], firewall_rules: [VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 7e30d6a5-06c5-48b0-a304-819497336a2b, kind: Instance { id: 60194fb6-a280-4030-a452-3f4a705fbb54 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 244, 103, 212])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(8395431), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: None, filter_protocols: Some([Icmp]), action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 7e30d6a5-06c5-48b0-a304-819497336a2b, kind: Instance { id: 60194fb6-a280-4030-a452-3f4a705fbb54 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 244, 103, 212])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(8395431), primary: true, slot: 0, transit_ips: [] }], filter_hosts: Some([Vpc(Vni(8395431))]), filter_ports: None, filter_protocols: None, action: Allow, priority: VpcFirewallRulePriority(65534) }, VpcFirewallRule { status: Enabled, direction: Inbound, targets: [NetworkInterface { id: 7e30d6a5-06c5-48b0-a304-819497336a2b, kind: Instance { id: 60194fb6-a280-4030-a452-3f4a705fbb54 }, name: Name(\"net0\"), ip: 172.30.0.5, mac: MacAddr(MacAddr6([168, 64, 37, 244, 103, 212])), subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), vni: Vni(8395431), primary: true, slot: 0, transit_ips: [] }], filter_hosts: None, filter_ports: Some([L4PortRange { first: L4Port(22), last: L4Port(22) }]), filter_protocols: Some([Tcp]), action: Allow, priority: VpcFirewallRulePriority(65534) }], dhcp_config: DhcpConfig { dns_servers: [1.1.1.1, 9.9.9.9], host_domain: None, search_domains: [] }, disks: [DiskRequest { device: \"nvme\", name: \"disk-95063f50cc0\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 3b001efd-ef58-42b1-9232-aa581e636831, read_only_parent: Some(Volume { block_size: 512, id: 9f47f0e9-08aa-463b-8505-23feaed489ca, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 4845401b-1c93-4371-965c-ef447b2b4c4e, key: Some(\"wBLcPlGM1lU0a4cNONAjXCEYfJYa8e7+N/aJipdWaCw=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::12]:19001\", \"[fd00:1122:3344:101::18]:19001\", \"[fd00:1122:3344:101::14]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 3b001efd-ef58-42b1-9232-aa581e636831, key: Some(\"a4ZH8kEQG4WXhxpUzSemMPaL2KbplBadaQMt3A1Xox8=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::12]:19002\", \"[fd00:1122:3344:101::14]:19002\", \"[fd00:1122:3344:101::19]:19000\"] } }] } }], cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAABnqYYFWQVZAACphgVZAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAZ6mGBVkFWQAAqYYFWQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiI2MDE5NGZiNi1hMjgwLTQwMzAtYTQ1Mi0zZjRhNzA1ZmJiNTQiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJQlJ3VnF1TjY1Z0lCYWwwc1NXUzBaQ01ia3dVRlJxY0tlNGFVVW1DZGRYMSJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\") }, instance_runtime: InstanceRuntimeState { propolis_id: Some(a99b853a-2530-488f-b74e-e3781d0d4d3c (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-08-05T16:53:18.731554843Z }, vmm_runtime: VmmRuntimeState { state: Starting, gen: Generation(1), time_updated: 2024-08-05T16:53:18.684161Z }, propolis_addr: [fd00:1122:3344:101::1:0]:12400 }","propolis_id":"a99b853a-2530-488f-b74e-e3781d0d4d3c","instance_id":"60194fb6-a280-4030-a452-3f4a705fbb54"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:19.073605936Z","hostname":"buskin","pid":1555,"uri":"/instances/60194fb6-a280-4030-a452-3f4a705fbb54","method":"PUT","req_id":"1045f688-f8f0-45b0-8aef-cd17816ef2cd","remote_addr":"[fd00:1122:3344:101::a]:63491","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":7508,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:19.208914094Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:37587"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:19.211443087Z","hostname":"buskin","pid":1555,"uri":"/instances/60194fb6-a280-4030-a452-3f4a705fbb54/state","method":"PUT","req_id":"5b98ab56-f6e5-46f1-94b7-8b4b83a2f6c3","remote_addr":"[fd00:1122:3344:101::a]:37587","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":2499,"response_code":"200"} {"msg":"Created OPTE port","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:19.216678116Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:426","port":"Port { inner: PortInner(PortData { name: \"opte6\", ip: 172.30.0.5, mac: MacAddr6([168, 64, 37, 244, 103, 212]), slot: 0, vni: Vni { inner: 8395431 }, subnet: V4(Ipv4Net { addr: 172.30.0.0, width: 22 }), gateway: Gateway { mac: MacAddr6([168, 64, 37, 255, 119, 119]), ip: 172.30.0.1 } }) }"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:19.216719497Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Configuring new Omicron zone: oxz_propolis-server_a99b853a-2530-488f-b74e-e3781d0d4d3c","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:19.25209752Z","hostname":"buskin","pid":1555,"instance_id":"60194fb6-a280-4030-a452-3f4a705fbb54","component":"InstanceManager","file":"illumos-utils/src/zone.rs:316"} {"msg":"Installing Omicron zone: oxz_propolis-server_a99b853a-2530-488f-b74e-e3781d0d4d3c","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:19.274562083Z","hostname":"buskin","pid":1555,"instance_id":"60194fb6-a280-4030-a452-3f4a705fbb54","component":"InstanceManager","file":"illumos-utils/src/zone.rs:354"} {"msg":"Profile for oxz_propolis-server_a99b853a-2530-488f-b74e-e3781d0d4d3c:\n\n\n \n \n \n \n \n \n \n \n \n \n \n","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:20.636386496Z","hostname":"buskin","pid":1555,"instance_id":"60194fb6-a280-4030-a452-3f4a705fbb54","component":"InstanceManager","file":"sled-agent/src/profile.rs:34"} {"msg":"Booting oxz_propolis-server_a99b853a-2530-488f-b74e-e3781d0d4d3c zone","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:20.636590819Z","hostname":"buskin","pid":1555,"zone":"oxz_propolis-server_a99b853a-2530-488f-b74e-e3781d0d4d3c","instance_id":"60194fb6-a280-4030-a452-3f4a705fbb54","component":"InstanceManager","file":"illumos-utils/src/running_zone.rs:472"} {"msg":"wait for service Some(\"oxz_propolis-server_a99b853a-2530-488f-b74e-e3781d0d4d3c\") failed: Property not found. retry in 51.567ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:53:20.815106333Z","hostname":"buskin","pid":1555,"zone":"oxz_propolis-server_a99b853a-2530-488f-b74e-e3781d0d4d3c","instance_id":"60194fb6-a280-4030-a452-3f4a705fbb54","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_a99b853a-2530-488f-b74e-e3781d0d4d3c\") failed: Property not found. retry in 137.460513ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:53:20.876692654Z","hostname":"buskin","pid":1555,"zone":"oxz_propolis-server_a99b853a-2530-488f-b74e-e3781d0d4d3c","instance_id":"60194fb6-a280-4030-a452-3f4a705fbb54","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_a99b853a-2530-488f-b74e-e3781d0d4d3c\") failed: Property not found. retry in 229.777213ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:53:21.038090286Z","hostname":"buskin","pid":1555,"zone":"oxz_propolis-server_a99b853a-2530-488f-b74e-e3781d0d4d3c","instance_id":"60194fb6-a280-4030-a452-3f4a705fbb54","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_a99b853a-2530-488f-b74e-e3781d0d4d3c\") failed: Property not found. retry in 404.130623ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:53:21.282546635Z","hostname":"buskin","pid":1555,"zone":"oxz_propolis-server_a99b853a-2530-488f-b74e-e3781d0d4d3c","instance_id":"60194fb6-a280-4030-a452-3f4a705fbb54","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_a99b853a-2530-488f-b74e-e3781d0d4d3c\") failed: Property not found. retry in 1.077818409s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:53:21.7009987Z","hostname":"buskin","pid":1555,"zone":"oxz_propolis-server_a99b853a-2530-488f-b74e-e3781d0d4d3c","instance_id":"60194fb6-a280-4030-a452-3f4a705fbb54","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_a99b853a-2530-488f-b74e-e3781d0d4d3c\") failed: Property not found. retry in 714.956478ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:53:22.79111142Z","hostname":"buskin","pid":1555,"zone":"oxz_propolis-server_a99b853a-2530-488f-b74e-e3781d0d4d3c","instance_id":"60194fb6-a280-4030-a452-3f4a705fbb54","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_a99b853a-2530-488f-b74e-e3781d0d4d3c\") failed: Property not found. retry in 1.431609109s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:53:23.515165728Z","hostname":"buskin","pid":1555,"zone":"oxz_propolis-server_a99b853a-2530-488f-b74e-e3781d0d4d3c","instance_id":"60194fb6-a280-4030-a452-3f4a705fbb54","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_a99b853a-2530-488f-b74e-e3781d0d4d3c\") failed: Property not found. retry in 1.118144274s","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:53:24.957634035Z","hostname":"buskin","pid":1555,"zone":"oxz_propolis-server_a99b853a-2530-488f-b74e-e3781d0d4d3c","instance_id":"60194fb6-a280-4030-a452-3f4a705fbb54","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Started propolis in zone: oxz_propolis-server_a99b853a-2530-488f-b74e-e3781d0d4d3c","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:26.105595222Z","hostname":"buskin","pid":1555,"instance_id":"60194fb6-a280-4030-a452-3f4a705fbb54","component":"InstanceManager","file":"sled-agent/src/instance.rs:1490"} {"msg":"wait for service Some(\"oxz_propolis-server_a99b853a-2530-488f-b74e-e3781d0d4d3c\") failed: Property not found. retry in 42.655881ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:53:26.114421442Z","hostname":"buskin","pid":1555,"instance_id":"60194fb6-a280-4030-a452-3f4a705fbb54","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_a99b853a-2530-488f-b74e-e3781d0d4d3c\") failed: Property not found. retry in 129.250035ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:53:26.166759605Z","hostname":"buskin","pid":1555,"instance_id":"60194fb6-a280-4030-a452-3f4a705fbb54","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"wait for service Some(\"oxz_propolis-server_a99b853a-2530-488f-b74e-e3781d0d4d3c\") failed: Property not found. retry in 213.493604ms","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:53:26.306956965Z","hostname":"buskin","pid":1555,"instance_id":"60194fb6-a280-4030-a452-3f4a705fbb54","component":"InstanceManager","file":"illumos-utils/src/svc.rs:36"} {"msg":"Propolis SMF service is online","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:26.530349255Z","hostname":"buskin","pid":1555,"instance_id":"60194fb6-a280-4030-a452-3f4a705fbb54","component":"InstanceManager","file":"sled-agent/src/instance.rs:1499"} {"msg":"Propolis HTTP server online","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:26.564995595Z","hostname":"buskin","pid":1555,"instance_id":"60194fb6-a280-4030-a452-3f4a705fbb54","component":"InstanceManager","file":"sled-agent/src/instance.rs:1513"} {"msg":"Sending ensure request to propolis: InstanceEnsureRequest { cloud_init_bytes: Some(\"6zyQTVNXSU40LjEAAgEBAAIAAioA+AEAKgBAAAAAAAAAAAAAAAApeFY0EmNpZGF0YSAgICAgRkFUMTIgICAOH75bfKwiwHQLVrQOuwcAzRBe6/Ay5M0WzRnr/lRoaXMgaXMgbm90IGEgYm9vdGFibGUgZGlzay4gIFBsZWFzZSBpbnNlcnQgYSBib290YWJsZSBmbG9wcHkgYW5kDQpwcmVzcyBhbnkga2V5IHRvIHRyeSBhZ2FpbiAuLi4gDQoAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAVar4////DwAAAAAAAAAA8P//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////D/j///8PAAAAAAAAAADw//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////8PY2lkYXRhICAgICAIAAAAAAAAAAAAAAAAAAAAAAAAAABBbQBlAHQAYQAtAA8A1GQAYQB0AGEAAAD//wAA/////01FVEEtRH4xICAgAABnqYYFWQVZAACphgVZAgC3AAAAQXUAcwBlAHIALQAPALBkAGEAdABhAAAA//8AAP////9VU0VSLUR+MSAgIAAAZ6mGBVkFWQAAqYYFWQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAHsiaW5zdGFuY2UtaWQiOiI2MDE5NGZiNi1hMjgwLTQwMzAtYTQ1Mi0zZjRhNzA1ZmJiNTQiLCJsb2NhbC1ob3N0bmFtZSI6ImxvY2Fsc2hhcmsiLCJwdWJsaWMta2V5cyI6WyJzc2gtZWQyNTUxOSBBQUFBQzNOemFDMWxaREkxTlRFNUFBQUFJQlJ3VnF1TjY1Z0lCYWwwc1NXUzBaQ01ia3dVRlJxY0tlNGFVVW1DZGRYMSJdfQAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA\"), disks: [DiskRequest { device: \"nvme\", name: \"disk-95063f50cc0\", read_only: false, slot: Slot(0), volume_construction_request: Volume { block_size: 512, id: 3b001efd-ef58-42b1-9232-aa581e636831, read_only_parent: Some(Volume { block_size: 512, id: 9f47f0e9-08aa-463b-8505-23feaed489ca, read_only_parent: None, sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 2, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 4845401b-1c93-4371-965c-ef447b2b4c4e, key: Some(\"wBLcPlGM1lU0a4cNONAjXCEYfJYa8e7+N/aJipdWaCw=\"), key_pem: None, lossy: false, read_only: true, root_cert_pem: None, target: [\"[fd00:1122:3344:101::12]:19001\", \"[fd00:1122:3344:101::18]:19001\", \"[fd00:1122:3344:101::14]:19001\"] } }] }), sub_volumes: [Region { block_size: 512, blocks_per_extent: 131072, extent_count: 32, gen: 1, opts: CrucibleOpts { cert_pem: None, control: None, flush_timeout: None, id: 3b001efd-ef58-42b1-9232-aa581e636831, key: Some(\"a4ZH8kEQG4WXhxpUzSemMPaL2KbplBadaQMt3A1Xox8=\"), key_pem: None, lossy: false, read_only: false, root_cert_pem: None, target: [\"[fd00:1122:3344:101::12]:19002\", \"[fd00:1122:3344:101::14]:19002\", \"[fd00:1122:3344:101::19]:19000\"] } }] } }], migrate: None, nics: [NetworkInterfaceRequest { name: \"opte6\", slot: Slot(0) }], properties: InstanceProperties { bootrom_id: 00000000-0000-0000-0000-000000000000, description: \"Test description\", id: 60194fb6-a280-4030-a452-3f4a705fbb54, image_id: 00000000-0000-0000-0000-000000000000, memory: 1024, metadata: InstanceMetadata { project_id: 19b9bd76-f1e4-448f-b46c-d56eb60fe928, silo_id: e179a288-a562-4f81-9527-488ea538b84b, sled_id: 3abf3a4f-8820-40af-a01f-4369b42a5529, sled_model: \"i86pc\", sled_revision: 0, sled_serial: \"buskin\" }, name: \"localshark\", vcpus: 2 } }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:26.565222069Z","hostname":"buskin","pid":1555,"instance_id":"60194fb6-a280-4030-a452-3f4a705fbb54","component":"InstanceManager","file":"sled-agent/src/instance.rs:736"} {"msg":"result of instance_ensure call is Ok(InstanceEnsureResponse { migrate: None })","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:26.704307851Z","hostname":"buskin","pid":1555,"instance_id":"60194fb6-a280-4030-a452-3f4a705fbb54","component":"InstanceManager","file":"sled-agent/src/instance.rs:738"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Creating), migration_status: NoMigration, time: 2024-08-05T16:53:26.706377216Z }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:26.706395207Z","hostname":"buskin","pid":1555,"instance_id":"60194fb6-a280-4030-a452-3f4a705fbb54","component":"InstanceManager","file":"sled-agent/src/instance.rs:622"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:26.706405307Z","hostname":"buskin","pid":1555,"instance_id":"60194fb6-a280-4030-a452-3f4a705fbb54","component":"InstanceManager","file":"sled-agent/src/instance.rs:644","new_vmm_state":"VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-08-05T16:53:26.706377216Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(a99b853a-2530-488f-b74e-e3781d0d4d3c (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-08-05T16:53:18.731554843Z }","propolis_id":"a99b853a-2530-488f-b74e-e3781d0d4d3c"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:26.706999037Z","hostname":"buskin","pid":1555,"instance_id":"60194fb6-a280-4030-a452-3f4a705fbb54","component":"InstanceManager","file":"sled-agent/src/instance.rs:541","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(a99b853a-2530-488f-b74e-e3781d0d4d3c (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-08-05T16:53:18.731554843Z }, propolis_id: a99b853a-2530-488f-b74e-e3781d0d4d3c (propolis), vmm_state: VmmRuntimeState { state: Starting, gen: Generation(2), time_updated: 2024-08-05T16:53:26.706377216Z }, migration_state: None }","instance_id":"60194fb6-a280-4030-a452-3f4a705fbb54"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Running), migration_status: NoMigration, time: 2024-08-05T16:53:26.774570929Z }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:26.77460139Z","hostname":"buskin","pid":1555,"instance_id":"60194fb6-a280-4030-a452-3f4a705fbb54","component":"InstanceManager","file":"sled-agent/src/instance.rs:622"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:26.77462608Z","hostname":"buskin","pid":1555,"instance_id":"60194fb6-a280-4030-a452-3f4a705fbb54","component":"InstanceManager","file":"sled-agent/src/instance.rs:644","new_vmm_state":"VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-08-05T16:53:26.774570929Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(a99b853a-2530-488f-b74e-e3781d0d4d3c (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-08-05T16:53:18.731554843Z }","propolis_id":"a99b853a-2530-488f-b74e-e3781d0d4d3c"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:26.774639151Z","hostname":"buskin","pid":1555,"instance_id":"60194fb6-a280-4030-a452-3f4a705fbb54","component":"InstanceManager","file":"sled-agent/src/instance.rs:541","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(a99b853a-2530-488f-b74e-e3781d0d4d3c (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-08-05T16:53:18.731554843Z }, propolis_id: a99b853a-2530-488f-b74e-e3781d0d4d3c (propolis), vmm_state: VmmRuntimeState { state: Running, gen: Generation(3), time_updated: 2024-08-05T16:53:26.774570929Z }, migration_state: None }","instance_id":"60194fb6-a280-4030-a452-3f4a705fbb54"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:26.810904869Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:52536"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:26.811634231Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"0c823a25-c65c-4202-b18f-370cc1384b70","remote_addr":"[fd00:1122:3344:101::c]:52536","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":698,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:26.854611364Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"dba5c045-d8b1-4638-995b-39ce1418bc5d","remote_addr":"[fd00:1122:3344:101::c]:52536","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":242,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:26.892097524Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:56208"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:26.892172135Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"f083fbdf-6fa1-4488-95e9-c47bae861893","remote_addr":"[fd00:1122:3344:101::c]:56208","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":35,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:26.904551226Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"0cbc9130-e917-40de-aba0-94fcf4104763","remote_addr":"[fd00:1122:3344:101::c]:56208","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":66,"response_code":"204"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:27.841177858Z","hostname":"buskin","pid":1555,"uri":"/3abf3a4f-8820-40af-a01f-4369b42a5529","method":"GET","req_id":"e2454d12-d00e-4903-8199-bd7df9a6bad1","remote_addr":"[fd00:1122:3344:101::d]:56611","local_addr":"[fd00:1122:3344:101::1]:40509","component":"dropshot","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","component":"producer-server","component":"MetricsManager","sled_id":"3abf3a4f-8820-40af-a01f-4369b42a5529","component":"SledAgent","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":534,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:31.427490515Z","hostname":"buskin","pid":1555,"local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:62740"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:31.427597827Z","hostname":"buskin","pid":1555,"uri":"/baseboard","method":"GET","req_id":"ba8b2c3d-59fd-49b1-98c8-f0bfa43fc7f1","remote_addr":"[fdb0:18c0:4d0d:9fb2::2]:62740","local_addr":"[fdb0:18c0:4d0d:9fb2::1]:80","component":"dropshot (BootstrapAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":83,"response_code":"200"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:35.541150505Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:59763"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:35.542117371Z","hostname":"buskin","pid":1555,"uri":"/instances/60194fb6-a280-4030-a452-3f4a705fbb54/state","method":"PUT","req_id":"66dc87f2-72e7-41c8-b414-71f6ada4d1de","remote_addr":"[fd00:1122:3344:101::a]:59763","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":921,"response_code":"200"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Stopping), migration_status: NoMigration, time: 2024-08-05T16:53:35.542223942Z }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:35.542246601Z","hostname":"buskin","pid":1555,"instance_id":"60194fb6-a280-4030-a452-3f4a705fbb54","component":"InstanceManager","file":"sled-agent/src/instance.rs:622"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:35.542259473Z","hostname":"buskin","pid":1555,"instance_id":"60194fb6-a280-4030-a452-3f4a705fbb54","component":"InstanceManager","file":"sled-agent/src/instance.rs:644","new_vmm_state":"VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-08-05T16:53:35.542223942Z }","new_instance_state":"InstanceRuntimeState { propolis_id: Some(a99b853a-2530-488f-b74e-e3781d0d4d3c (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-08-05T16:53:18.731554843Z }","propolis_id":"a99b853a-2530-488f-b74e-e3781d0d4d3c"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:35.542279751Z","hostname":"buskin","pid":1555,"instance_id":"60194fb6-a280-4030-a452-3f4a705fbb54","component":"InstanceManager","file":"sled-agent/src/instance.rs:541","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: Some(a99b853a-2530-488f-b74e-e3781d0d4d3c (propolis)), dst_propolis_id: None, migration_id: None, gen: Generation(3), time_updated: 2024-08-05T16:53:18.731554843Z }, propolis_id: a99b853a-2530-488f-b74e-e3781d0d4d3c (propolis), vmm_state: VmmRuntimeState { state: Stopping, gen: Generation(5), time_updated: 2024-08-05T16:53:35.542223942Z }, migration_state: None }","instance_id":"60194fb6-a280-4030-a452-3f4a705fbb54"} {"msg":"Observing new propolis state: ObservedPropolisState { vmm_state: PropolisInstanceState(Destroyed), migration_status: NoMigration, time: 2024-08-05T16:53:35.574476623Z }","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:35.574498083Z","hostname":"buskin","pid":1555,"instance_id":"60194fb6-a280-4030-a452-3f4a705fbb54","component":"InstanceManager","file":"sled-agent/src/instance.rs:622"} {"msg":"updated state after observing Propolis state change","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:35.574519993Z","hostname":"buskin","pid":1555,"instance_id":"60194fb6-a280-4030-a452-3f4a705fbb54","component":"InstanceManager","file":"sled-agent/src/instance.rs:644","new_vmm_state":"VmmRuntimeState { state: Destroyed, gen: Generation(6), time_updated: 2024-08-05T16:53:35.574476623Z }","new_instance_state":"InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-08-05T16:53:35.574476623Z }","propolis_id":"a99b853a-2530-488f-b74e-e3781d0d4d3c"} {"msg":"terminating VMM that has exited","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:35.574559934Z","hostname":"buskin","pid":1555,"instance_id":"60194fb6-a280-4030-a452-3f4a705fbb54","component":"InstanceManager","file":"sled-agent/src/instance.rs:659","instance_id":"60194fb6-a280-4030-a452-3f4a705fbb54"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:35.574567714Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"Received GetLatestResources(...)","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:35.574750337Z","hostname":"buskin","pid":1555,"component":"StorageManager","file":"sled-storage/src/manager.rs:405"} {"msg":"creating zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:35.574795338Z","hostname":"buskin","pid":1555,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:445","context":"ZoneBundleContext { storage_dirs: [\"/pool/int/4576fedf-479b-43b3-8fad-bb8b7ec274f8/debug/bundle/zone\", \"/pool/int/d09b3f64-0a36-4052-8233-a94e62e662e2/debug/bundle/zone\"], cause: TerminatedInstance, extra_log_dirs: [\"/pool/ext/bd033efe-47fe-4ba2-8bc4-ba99e628ada7/crypt/debug\", \"/pool/ext/1aa678c2-e32d-4428-a115-7e698777172b/crypt/debug\", \"/pool/ext/cf764ad4-ee9a-4ce5-bd3d-f5a7f6798396/crypt/debug\", \"/pool/ext/939962e4-70b1-47be-b2d6-938c818b62c4/crypt/debug\", \"/pool/ext/52c33af8-e5fc-4cb6-93db-7a529f23be77/crypt/debug\", \"/pool/ext/cb14544a-cc03-4a04-84a7-da1d51f0a71d/crypt/debug\", \"/pool/ext/2cb76428-77cf-4416-ab3e-f66fc49a03b4/crypt/debug\", \"/pool/ext/1a42fe8d-3436-46f3-8051-0d2ff2a0be4f/crypt/debug\", \"/pool/ext/85b774a7-53a2-4f22-a027-1f642ba2d7f9/crypt/debug\"] }","zone_name":"oxz_propolis-server_a99b853a-2530-488f-b74e-e3781d0d4d3c"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:35.611234049Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:58434"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:35.612064014Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"c88b249d-618b-47a2-a623-9fbd9c4d026f","remote_addr":"[fd00:1122:3344:101::c]:58434","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":748,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:35.623386447Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"d2aba864-f7d2-4c6c-b55b-01a497d78a36","remote_addr":"[fd00:1122:3344:101::c]:58434","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":135,"response_code":"204"} {"msg":"finished zone bundle","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:36.397151752Z","hostname":"buskin","pid":1555,"component":"ZoneBundler","file":"sled-agent/src/zone_bundle.rs:1249","metadata":"ZoneBundleMetadata { id: ZoneBundleId { zone_name: \"oxz_propolis-server_a99b853a-2530-488f-b74e-e3781d0d4d3c\", bundle_id: e4e01b87-032c-44b4-8c80-8eec8f3a2869 }, time_created: 2024-08-05T16:53:35.575014722Z, version: 0, cause: TerminatedInstance }"} {"msg":"Halting and removing zone: oxz_propolis-server_a99b853a-2530-488f-b74e-e3781d0d4d3c","v":0,"name":"SledAgent","level":40,"time":"2024-08-05T16:53:36.397232653Z","hostname":"buskin","pid":1555,"instance_id":"60194fb6-a280-4030-a452-3f4a705fbb54","component":"InstanceManager","file":"sled-agent/src/instance.rs:831"} {"msg":"halt_and_remove_logged: Previous zone state: Running","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:38.169810181Z","hostname":"buskin","pid":1555,"instance_id":"60194fb6-a280-4030-a452-3f4a705fbb54","component":"InstanceManager","file":"illumos-utils/src/zone.rs:270"} {"msg":"Removed route set for subnet","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:38.185550079Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:906","id":"RouterId { vni: Vni(8395431), kind: System }"} {"msg":"Removed route set for subnet","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:38.18560893Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:906","id":"RouterId { vni: Vni(8395431), kind: Custom(V4(Ipv4Net { addr: 172.30.0.0, width: 22 })) }"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:38.186121379Z","hostname":"buskin","pid":1555,"instance_id":"60194fb6-a280-4030-a452-3f4a705fbb54","component":"InstanceManager","file":"sled-agent/src/instance.rs:541","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-08-05T16:53:35.574476623Z }, propolis_id: a99b853a-2530-488f-b74e-e3781d0d4d3c (propolis), vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-08-05T16:53:38.186089998Z }, migration_state: None }","instance_id":"60194fb6-a280-4030-a452-3f4a705fbb54"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:38.308056308Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:51985"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:38.308352983Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"33e43e8e-c037-4a04-88ed-235e374e12ea","remote_addr":"[fd00:1122:3344:101::c]:51985","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":242,"response_code":"200"} {"msg":"Mapping virtual NIC to physical host","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:38.308658848Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:780","mapping":"VirtualNetworkInterfaceHost { virtual_ip: 172.30.0.5, virtual_mac: MacAddr(MacAddr6([168, 64, 37, 244, 103, 212])), physical_host_ip: fd00:1122:3344:101::1, vni: Vni(8395431) }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:38.308696099Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"PUT","req_id":"2f35e687-f8c6-42c1-935f-45ffcaf075bf","remote_addr":"[fd00:1122:3344:101::c]:51985","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":123,"response_code":"204"} {"msg":"Publishing instance state update to Nexus","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:38.672107776Z","hostname":"buskin","pid":1555,"instance_id":"60194fb6-a280-4030-a452-3f4a705fbb54","component":"InstanceManager","file":"sled-agent/src/instance.rs:541","state":"SledInstanceState { instance_state: InstanceRuntimeState { propolis_id: None, dst_propolis_id: None, migration_id: None, gen: Generation(5), time_updated: 2024-08-05T16:53:35.574476623Z }, propolis_id: a99b853a-2530-488f-b74e-e3781d0d4d3c (propolis), vmm_state: VmmRuntimeState { state: Destroyed, gen: Generation(7), time_updated: 2024-08-05T16:53:38.186089998Z }, migration_state: None }","instance_id":"60194fb6-a280-4030-a452-3f4a705fbb54"} {"msg":"State monitoring task complete","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:38.676213986Z","hostname":"buskin","pid":1555,"instance_id":"60194fb6-a280-4030-a452-3f4a705fbb54","component":"InstanceManager","file":"sled-agent/src/instance.rs:774"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:38.709363501Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:57773"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:38.709464313Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"96049b85-91aa-4440-86c2-abfce036a133","remote_addr":"[fd00:1122:3344:101::c]:57773","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":52,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:38.715826182Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"0f2cf1b7-75c8-4d00-b626-c46a1a7a90c6","remote_addr":"[fd00:1122:3344:101::c]:57773","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":94,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:38.756947533Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::c]:58512"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:38.757017814Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"b9b289a9-1efc-4a3c-a34a-089b3db8230c","remote_addr":"[fd00:1122:3344:101::c]:58512","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":40,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:38.762956315Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"02eb465f-55f8-4160-99de-a62ab96c3b0e","remote_addr":"[fd00:1122:3344:101::c]:58512","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":46,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:39.268968183Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:34318"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:39.269192019Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"GET","req_id":"a18977f6-b66f-4957-ae94-83b3cb17b8e7","remote_addr":"[fd00:1122:3344:101::a]:34318","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":146,"response_code":"200"} {"msg":"Clearing mapping of virtual NIC to physical host","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:39.269427583Z","hostname":"buskin","pid":1555,"component":"PortManager","file":"illumos-utils/src/opte/port_manager.rs:820","mapping":"VirtualNetworkInterfaceHost { virtual_ip: 172.30.0.5, virtual_mac: MacAddr(MacAddr6([168, 64, 37, 244, 103, 212])), physical_host_ip: fd00:1122:3344:101::1, vni: Vni(8395431) }"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:39.269598255Z","hostname":"buskin","pid":1555,"uri":"/v2p","method":"DELETE","req_id":"6fb45b3a-80f4-4353-bcb4-de97a1f03c87","remote_addr":"[fd00:1122:3344:101::a]:34318","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":215,"response_code":"204"} {"msg":"accepted connection","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:41.346600064Z","hostname":"buskin","pid":1555,"local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:775","remote_addr":"[fd00:1122:3344:101::a]:51735"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:41.346675266Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"GET","req_id":"2471c248-3ba5-4df0-a241-bad3bf9a4467","remote_addr":"[fd00:1122:3344:101::a]:51735","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":65,"response_code":"200"} {"msg":"request completed","v":0,"name":"SledAgent","level":30,"time":"2024-08-05T16:53:41.357985999Z","hostname":"buskin","pid":1555,"uri":"/vpc-routes","method":"PUT","req_id":"1d65cc02-c9f6-4272-b0ad-a9aaf3fcc039","remote_addr":"[fd00:1122:3344:101::a]:51735","local_addr":"[fd00:1122:3344:101::1]:12345","component":"dropshot (SledAgent)","file":"/home/build/.cargo/git/checkouts/dropshot-a4a923d29dccc492/0c0bae3/dropshot/src/server.rs:914","latency_us":116,"response_code":"204"}